diff options
| author | Georgios Pinitas <georgios.pinitas@arm.com> | 2019-10-14 19:03:09 +0100 |
|---|---|---|
| committer | Georgios Pinitas <georgios.pinitas@arm.com> | 2019-10-23 12:08:12 +0000 |
| commit | 48b3ef89de5f21a0169d8416e3d54081f82c7bf8 (patch) | |
| tree | f857d733ccf446c704823dc7ac796a96eb55095e /src/core/NEON/kernels/arm_gemm/kernels | |
| parent | 1dce3101ef8d77c8cf0af7dfd4af6595a0136b91 (diff) | |
| download | ComputeLibrary-48b3ef89de5f21a0169d8416e3d54081f82c7bf8.tar.gz | |
COMPMID-2577: Fuse bias addition and activation in gemm assembly kernels
Change-Id: I7f52112d2d05b1ea3d3f3d4b19b8eafab05d6c44
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Reviewed-on: https://review.mlplatform.org/c/2141
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Marquez <pablo.tello@arm.com>
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/kernels')
59 files changed, 32404 insertions, 32598 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp index cc205dc6e3..941281f24d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp @@ -61,7 +61,9 @@ public: kern_type kernel = a64_gemm_s8_12x8; gemm_s8_12x8(const CPUInfo *ci) { - if (ci->get_cpu_model() == CPUModel::A55r1) { + auto mod = ci->get_cpu_model(); + + if (mod == CPUModel::A55r1) { kernel = a64_gemm_s8_12x8_a55r1; } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp index 9032ba67b3..c228b8b858 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp @@ -25,8 +25,6 @@ #ifdef __aarch64__ -#include "arm_gemm.hpp" - #include "../std_transforms_fixed.hpp" namespace arm_gemm { @@ -71,7 +69,9 @@ public: kern_type kernel = a64_gemm_u8_12x8; gemm_u8_12x8(const CPUInfo *ci) { - if (ci->get_cpu_model() == CPUModel::A55r1) { + auto mod = ci->get_cpu_model(); + + if (mod == CPUModel::A55r1) { kernel = a64_gemm_u8_12x8_a55r1; } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp index 5b850b7a20..283fea1102 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp @@ -25,8 +25,6 @@ #if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)) -#include "arm_gemm.hpp" - #include "../std_transforms_fixed.hpp" namespace arm_gemm { @@ -66,7 +64,9 @@ public: kern_type kernel = a64_hgemm_asimd_24x8; hgemm_24x8(const CPUInfo *ci) { - if (ci->get_cpu_model() == CPUModel::A55r1) { + auto model = ci->get_cpu_model(); + + if (model == CPUModel::A55r1) { kernel = a64_hgemm_asimd_24x8_a55r1; } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp index 560593958a..1ce934d413 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_hybrid_fp32_mla_16x4(const float *, int, const float *, float *, int, float, int, int, int); -void a64_hybrid_fp32_mla_16x4_a55(const float *, int, const float *, float *, int, float, int, int, int); +void a64_hybrid_fp32_mla_16x4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); +void a64_hybrid_fp32_mla_16x4_a55(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); class hybrid_fp32_mla_16x4 { @@ -41,10 +41,10 @@ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -54,11 +54,26 @@ public: return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + StdTransformsFixed<operand_type, result_type, 4, 16, 1> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp index 7261761d7e..e3325f7728 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp @@ -25,20 +25,41 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long blocks_count = K / 1; + float nullbias[16]; + if (!append && !bias) { + memset(nullbias, 0, (16 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const float * const a_ptr0_base = A + (y * lda); @@ -48,7 +69,6 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -58,7 +78,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float float result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(float); float *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -68,6 +88,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float if (use_result_buffer) { c_ptr0 = result_buffer; } + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -76,17 +97,17 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "temploadreg1 .req X1\n" "temploadreg2 .req X2\n" "temploadreg3 .req X3\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -95,21 +116,16 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v17.4s, v17.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -434,6 +450,16 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v19.4s, v11.4s, v0.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -444,7 +470,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -458,56 +484,47 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "temploadreg3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -931,9 +948,27 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v23.4s, v11.4s, v1.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" "str q20, [c_ptr1]\n" @@ -947,7 +982,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; @@ -965,131 +1000,118 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v24.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v25.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "movi v26.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "movi v27.4s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ins v14.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "ins v14.d[1], temploadreg2\n" "cbz %[loops], 2f\n" "3:\n" "fmla v16.4s, v8.4s, v0.s[0]\n" - "ldr d15, [%[b_ptr0], #-0x10]\n" + "ins v14.d[1], temploadreg2\n" "fmla v20.4s, v8.4s, v1.s[0]\n" - "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "ldr d15, [%[b_ptr0], #-0x10]\n" "fmla v24.4s, v8.4s, v2.s[0]\n" - "ldr d4, [%[a_ptr0]]\n" + "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "fmla v17.4s, v9.4s, v0.s[0]\n" - "ldr temploadreg0, [%[a_ptr0], #0x8]\n" + "ldr d4, [%[a_ptr0]]\n" "fmla v21.4s, v9.4s, v1.s[0]\n" - "ldr d5, [a_ptr1]\n" + "ldr temploadreg0, [%[a_ptr0], #0x8]\n" "fmla v25.4s, v9.4s, v2.s[0]\n" - "ldr temploadreg1, [a_ptr1, #0x8]\n" + "ldr d5, [a_ptr1]\n" "fmla v18.4s, v10.4s, v0.s[0]\n" - "ldr d6, [a_ptr2]\n" + "ldr temploadreg1, [a_ptr1, #0x8]\n" "fmla v22.4s, v10.4s, v1.s[0]\n" - "ldr temploadreg2, [a_ptr2, #0x8]\n" + "ldr d6, [a_ptr2]\n" "fmla v26.4s, v10.4s, v2.s[0]\n" - "ldr d8, [%[b_ptr0]]\n" + "ldr temploadreg2, [a_ptr2, #0x8]\n" "fmla v19.4s, v11.4s, v0.s[0]\n" - "ins v4.d[1], temploadreg0\n" + "ldr d8, [%[b_ptr0]]\n" "fmla v23.4s, v11.4s, v1.s[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ins v4.d[1], temploadreg0\n" "fmla v27.4s, v11.4s, v2.s[0]\n" - "ldr d9, [%[b_ptr0], #0x10]\n" + "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "fmla v16.4s, v12.4s, v0.s[1]\n" - "ins v5.d[1], temploadreg1\n" + "ldr d9, [%[b_ptr0], #0x10]\n" "fmla v20.4s, v12.4s, v1.s[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ins v5.d[1], temploadreg1\n" "fmla v24.4s, v12.4s, v2.s[1]\n" - "ldr d10, [%[b_ptr0], #0x20]\n" + "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "fmla v17.4s, v13.4s, v0.s[1]\n" - "ins v6.d[1], temploadreg2\n" + "ldr d10, [%[b_ptr0], #0x20]\n" "fmla v21.4s, v13.4s, v1.s[1]\n" - "ldr temploadreg2, [%[b_ptr0], #0x28]\n" + "ins v6.d[1], temploadreg2\n" "fmla v25.4s, v13.4s, v2.s[1]\n" - "ldr d11, [%[b_ptr0], #0x30]\n" + "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "fmla v18.4s, v14.4s, v0.s[1]\n" - "ins v15.d[1], temploadreg3\n" + "ldr d11, [%[b_ptr0], #0x30]\n" "fmla v22.4s, v14.4s, v1.s[1]\n" - "ldr temploadreg3, [%[b_ptr0], #0x38]\n" + "ins v15.d[1], temploadreg3\n" "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - "ins v8.d[1], temploadreg0\n" "subs %[loops], %[loops], #0x1\n" "fmla v19.4s, v15.4s, v0.s[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x48]\n" + "ins v8.d[1], temploadreg0\n" "fmla v23.4s, v15.4s, v1.s[1]\n" - "ldr d13, [%[b_ptr0], #0x50]\n" + "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" "fmla v16.4s, v8.4s, v0.s[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "fmla v20.4s, v8.4s, v1.s[2]\n" @@ -1113,41 +1135,39 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v23.4s, v11.4s, v1.s[2]\n" "ins v15.d[1], temploadreg3\n" "fmla v27.4s, v11.4s, v2.s[2]\n" - "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - "fmla v16.4s, v12.4s, v0.s[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" "ins v8.d[1], temploadreg0\n" - "fmla v27.4s, v15.4s, v2.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" + "ins v9.d[1], temploadreg1\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla v16.4s, v8.4s, v4.s[0]\n" - "ins v9.d[1], temploadreg1\n" - "fmla v20.4s, v8.4s, v5.s[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - "fmla v24.4s, v8.4s, v6.s[0]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" "ins v10.d[1], temploadreg2\n" - "add a_ptr1, a_ptr1, #0x20\n" "fmla v17.4s, v9.4s, v4.s[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "fmla v21.4s, v9.4s, v5.s[0]\n" @@ -1163,70 +1183,71 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v19.4s, v11.4s, v4.s[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" "fmla v23.4s, v11.4s, v5.s[0]\n" - "ldr d1, [a_ptr1, #-0x10]\n" - "fmla v27.4s, v11.4s, v6.s[0]\n" "ins v13.d[1], temploadreg1\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "ins v14.d[1], temploadreg2\n" "fmla v16.4s, v12.4s, v4.s[1]\n" - "ldr temploadreg1, [a_ptr1, #-0x8]\n" + "ldr d8, [%[b_ptr0]]\n" "fmla v20.4s, v12.4s, v5.s[1]\n" - "ins v14.d[1], temploadreg2\n" + "ins v0.d[1], temploadreg0\n" "fmla v24.4s, v12.4s, v6.s[1]\n" - "ldr d8, [%[b_ptr0]]\n" + "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "fmla v17.4s, v13.4s, v4.s[1]\n" - "ins v0.d[1], temploadreg0\n" + "ldr d9, [%[b_ptr0], #0x10]\n" "fmla v21.4s, v13.4s, v5.s[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ldr d10, [%[b_ptr0], #0x20]\n" "fmla v25.4s, v13.4s, v6.s[1]\n" - "ldr d9, [%[b_ptr0], #0x10]\n" + "ldr d11, [%[b_ptr0], #0x30]\n" "fmla v18.4s, v14.4s, v4.s[1]\n" - "ins v1.d[1], temploadreg1\n" + "ins v15.d[1], temploadreg3\n" "fmla v22.4s, v14.4s, v5.s[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "fmla v26.4s, v14.4s, v6.s[1]\n" - "ldr d10, [%[b_ptr0], #0x20]\n" - "ldr d11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x20\n" - "ins v15.d[1], temploadreg3\n" - "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - "ldr d2, [a_ptr2, #-0x10]\n" - "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "ldr d12, [%[b_ptr0], #0x40]\n" + "ins v8.d[1], temploadreg0\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla v19.4s, v15.4s, v4.s[1]\n" - "ldr temploadreg2, [a_ptr2, #-0x8]\n" + "ldr d1, [a_ptr1, #-0x10]\n" "fmla v23.4s, v15.4s, v5.s[1]\n" - "ldr temploadreg3, [%[b_ptr0], #0x38]\n" + "ldr temploadreg1, [a_ptr1, #-0x8]\n" "fmla v27.4s, v15.4s, v6.s[1]\n" - "ldr d12, [%[b_ptr0], #0x40]\n" - "ins v8.d[1], temploadreg0\n" - "ins v2.d[1], temploadreg2\n" - "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "fmla v16.4s, v8.4s, v4.s[2]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "fmla v20.4s, v8.4s, v5.s[2]\n" - "ins v9.d[1], temploadreg1\n" + "ins v1.d[1], temploadreg1\n" "fmla v24.4s, v8.4s, v6.s[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x58]\n" + "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "ins v10.d[1], temploadreg2\n" - "fmla v17.4s, v9.4s, v4.s[2]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "fmla v21.4s, v9.4s, v5.s[2]\n" + "add a_ptr2, a_ptr2, #0x20\n" "ldr d15, [%[b_ptr0], #0x70]\n" - "fmla v25.4s, v9.4s, v6.s[2]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "ldr d2, [a_ptr2, #-0x10]\n" + "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "ldr temploadreg2, [a_ptr2, #-0x8]\n" + "ins v9.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ins v11.d[1], temploadreg3\n" - "fmla v18.4s, v10.4s, v4.s[2]\n" + "ins v2.d[1], temploadreg2\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "ldr temploadreg2, [%[b_ptr0], #0x28]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" "ins v12.d[1], temploadreg0\n" - "fmla v26.4s, v10.4s, v6.s[2]\n" - "ins v13.d[1], temploadreg1\n" "fmla v19.4s, v11.4s, v4.s[2]\n" - "ins v14.d[1], temploadreg2\n" + "ins v10.d[1], temploadreg2\n" "fmla v23.4s, v11.4s, v5.s[2]\n" - "ins v15.d[1], temploadreg3\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "fmla v27.4s, v11.4s, v6.s[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x100\n" + "ins v13.d[1], temploadreg1\n" "fmla v16.4s, v12.4s, v4.s[3]\n" + "ins v15.d[1], temploadreg3\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "ins v14.d[1], temploadreg2\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x100\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "fmla v20.4s, v12.4s, v5.s[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" @@ -1258,13 +1279,13 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" - "ins v14.d[1], temploadreg2\n" "b.ne 3b\n" "2:\n" - "ldr d15, [%[b_ptr0], #-0x10]\n" + "ins v14.d[1], temploadreg2\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" - "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [c_ptr1]\n" + "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" @@ -1554,13 +1575,39 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v27.4s, v11.4s, v2.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" "str q22, [c_ptr1, #0x20]\n" "str q23, [c_ptr1, #0x30]\n" "str q24, [c_ptr2]\n" @@ -1576,7 +1623,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory" ); break; @@ -1599,89 +1646,72 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q3, [a_ptr3]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v24.4s, #0\n" + "mov v28.16b, v16.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v25.4s, #0\n" + "mov v29.16b, v17.16b\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v26.4s, #0\n" + "mov v30.16b, v18.16b\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "movi v27.4s, #0\n" + "mov v31.16b, v19.16b\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "movi v28.4s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "movi v29.4s, #0\n" - "ins v14.d[1], temploadreg2\n" - "movi v30.4s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "movi v31.4s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" + "ins v14.d[1], temploadreg2\n" "add a_ptr3, a_ptr3, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "fmul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "fmul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "fmul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "fmul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -2303,16 +2333,50 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float "fmla v31.4s, v11.4s, v3.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v14.4s\n" + "fmax v29.4s, v29.4s, v14.4s\n" "str q22, [c_ptr1, #0x20]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" "str q23, [c_ptr1, #0x30]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" "str q24, [c_ptr2]\n" + "fmin v31.4s, v31.4s, v15.4s\n" "str q25, [c_ptr2, #0x10]\n" "str q26, [c_ptr2, #0x20]\n" "str q27, [c_ptr2, #0x30]\n" @@ -2331,7 +2395,7 @@ void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp index 504769b9f0..d60d08a672 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp @@ -25,20 +25,41 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long blocks_count = K / 1; + float nullbias[16]; + if (!append && !bias) { + memset(nullbias, 0, (16 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const float * const a_ptr0_base = A + (y * lda); @@ -48,7 +69,6 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -58,7 +78,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, float result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(float); float *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -68,21 +88,22 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, if (use_result_buffer) { c_ptr0 = result_buffer; } + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: __asm __volatile ( - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -90,21 +111,16 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v17.4s, v17.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -291,13 +307,23 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v19.4s, v11.4s, v0.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" ); break; @@ -307,55 +333,46 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "c_ptr1 .req X1\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -635,9 +652,27 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v23.4s, v11.4s, v1.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" "str q20, [c_ptr1]\n" @@ -647,7 +682,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" ); break; @@ -661,71 +696,58 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v24.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v25.4s, #0\n" - "ldr q14, [%[b_ptr0], #0x60]\n" - "movi v26.4s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "movi v27.4s, #0\n" + "ldr q14, [%[b_ptr0], #0x60]\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -1098,13 +1120,39 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v27.4s, v11.4s, v2.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" "str q22, [c_ptr1, #0x20]\n" "str q23, [c_ptr1, #0x30]\n" "str q24, [c_ptr2]\n" @@ -1116,7 +1164,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1135,87 +1183,70 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" - "movi v16.4s, #0\n" + "cbnz %[append], 1f\n" + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v20.16b, v16.16b\n" "ldr q0, [%[a_ptr0]]\n" - "movi v17.4s, #0\n" + "mov v21.16b, v17.16b\n" "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" + "mov v22.16b, v18.16b\n" "ldr q2, [a_ptr2]\n" - "movi v19.4s, #0\n" + "mov v23.16b, v19.16b\n" "ldr q3, [a_ptr3]\n" - "movi v20.4s, #0\n" + "mov v24.16b, v16.16b\n" "ldr q8, [%[b_ptr0]]\n" - "movi v21.4s, #0\n" + "mov v25.16b, v17.16b\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v22.4s, #0\n" + "mov v26.16b, v18.16b\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v23.4s, #0\n" + "mov v27.16b, v19.16b\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v24.4s, #0\n" + "mov v28.16b, v16.16b\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v25.4s, #0\n" + "mov v29.16b, v17.16b\n" "ldr q13, [%[b_ptr0], #0x50]\n" - "movi v26.4s, #0\n" + "mov v30.16b, v18.16b\n" "ldr q14, [%[b_ptr0], #0x60]\n" - "movi v27.4s, #0\n" + "mov v31.16b, v19.16b\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "movi v28.4s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "movi v29.4s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "movi v30.4s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "fmul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "fmul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "fmul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "fmul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "fmul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "fmul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "fmul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "fmul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "fmul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "fmul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "fmul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "fmul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "fmul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "fmul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "fmul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "fmul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" @@ -1681,16 +1712,50 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, "fmla v31.4s, v11.4s, v3.s[0]\n" "b.ne 7b\n" "6:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v14.4s\n" + "fmax v29.4s, v29.4s, v14.4s\n" "str q22, [c_ptr1, #0x20]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" "str q23, [c_ptr1, #0x30]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" "str q24, [c_ptr2]\n" + "fmin v31.4s, v31.4s, v15.4s\n" "str q25, [c_ptr2, #0x10]\n" "str q26, [c_ptr2, #0x20]\n" "str q27, [c_ptr2, #0x30]\n" @@ -1705,7 +1770,7 @@ void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp index c8934dff8a..5a6fabcfa9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_hybrid_s8s32_dot_16x4(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); -void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void a64_hybrid_s8s32_dot_16x4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); +void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class hybrid_s8s32_dot_16x4 { @@ -41,10 +41,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -54,11 +54,26 @@ public: return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp index 3f72e4bea1..3ecf0151aa 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -60,7 +62,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in int32_t result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(int32_t); int32_t *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -78,7 +80,6 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X1\n" "temploadreg2 .req X2\n" "temploadreg3 .req X3\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -94,44 +95,21 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr d14, [%[b_ptr0], #0x60]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q10, [%[b_ptr0], #0x20]\n" - "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "subs %[loops], %[loops], #0x1\n" @@ -146,108 +124,108 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" "ldr d8, [%[b_ptr0]]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" @@ -258,29 +236,29 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ins v14.d[1], temploadreg2\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" @@ -291,137 +269,137 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -430,30 +408,30 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -464,7 +442,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -476,178 +454,144 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X3\n" "temploadreg2 .req X4\n" "temploadreg3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" + "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "movi v20.4s, #0\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "movi v21.4s, #0\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" + "movi v22.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" + "movi v23.4s, #0\n" "ldr d14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ins v14.d[1], temploadreg2\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - "subs %[loops], %[loops], #0x1\n" "ldr d11, [%[b_ptr0], #0x30]\n" - "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "subs %[loops], %[loops], #0x1\n" "ins v15.d[1], temploadreg3\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "add a_ptr1, a_ptr1, #0x20\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v1.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" @@ -656,55 +600,55 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -714,223 +658,223 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" - "b.ne 3b\n" - "2:\n" "ins v14.d[1], temploadreg2\n" - "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "b.ne 2b\n" + "1:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - "prfm PSTL1KEEP, [c_ptr1]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr d13, [%[b_ptr0], #0x50]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -939,43 +883,43 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -992,7 +936,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; @@ -1006,228 +950,183 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X5\n" "temploadreg2 .req X6\n" "temploadreg3 .req X7\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "movi v25.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v26.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" + "ldr q1, [a_ptr1]\n" "movi v27.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" "ins v14.d[1], temploadreg2\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "ins v14.d[1], temploadreg2\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x20\n" @@ -1235,65 +1134,65 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" "ldr d2, [a_ptr2, #-0x10]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ins v2.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -1304,275 +1203,275 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1581,56 +1480,56 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1653,7 +1552,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory" ); break; @@ -1670,364 +1569,309 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X7\n" "temploadreg2 .req X8\n" "temploadreg3 .req X9\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q3, [a_ptr3]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "movi v25.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v26.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" + "ldr q1, [a_ptr1]\n" "movi v27.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" + "ins v14.d[1], temploadreg2\n" "movi v28.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add a_ptr2, a_ptr1, %[lda]\n" "movi v29.4s, #0\n" - "ins v14.d[1], temploadreg2\n" + "ldr q2, [a_ptr2]\n" "movi v30.4s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" + "add a_ptr3, a_ptr2, %[lda]\n" "movi v31.4s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add a_ptr3, a_ptr3, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" - "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" - "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" "ldr q3, [a_ptr3]\n" - "ldr q8, [%[b_ptr0]]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr3, a_ptr3, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr d14, [%[b_ptr0], #0x60]\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "ins v14.d[1], temploadreg2\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr d2, [a_ptr2, #-0x10]\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ins v2.d[1], temploadreg2\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr d3, [a_ptr3, #-0x10]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" "ldr temploadreg3, [a_ptr3, #-0x8]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ins v3.d[1], temploadreg3\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" "ins v14.d[1], temploadreg2\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" @@ -2035,319 +1879,319 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -2356,69 +2200,69 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "ld1 {v3.b}[0], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "ld1 {v3.b}[1], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" "ld1 {v3.b}[2], [a_ptr3]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -2447,7 +2291,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp index 1ad924612e..b48b674621 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -60,7 +62,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ int32_t result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(int32_t); int32_t *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -74,7 +76,6 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ switch(M-y) { case 1: __asm __volatile ( - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -89,196 +90,174 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q10, [%[b_ptr0], #0x20]\n" - "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -287,37 +266,37 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" ); break; @@ -325,316 +304,283 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ __asm __volatile ( "a_ptr1 .req X0\n" "c_ptr1 .req X1\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" - "movi v18.4s, #0\n" "ldr q8, [%[b_ptr0]]\n" - "movi v19.4s, #0\n" + "movi v18.4s, #0\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "movi v20.4s, #0\n" + "movi v19.4s, #0\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "movi v21.4s, #0\n" + "movi v20.4s, #0\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "movi v22.4s, #0\n" + "movi v21.4s, #0\n" "ldr q12, [%[b_ptr0], #0x40]\n" - "movi v23.4s, #0\n" + "movi v22.4s, #0\n" "ldr q13, [%[b_ptr0], #0x50]\n" + "movi v23.4s, #0\n" "ldr q14, [%[b_ptr0], #0x60]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q9, [%[b_ptr0], #0x10]\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b.ne 3b\n" - "2:\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -643,43 +589,43 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -692,7 +638,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" ); break; @@ -702,421 +648,378 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "a_ptr2 .req X1\n" "c_ptr1 .req X2\n" "c_ptr2 .req X3\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v25.4s, #0\n" - "ldr q14, [%[b_ptr0], #0x60]\n" + "ldr q1, [a_ptr1]\n" "movi v26.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add a_ptr2, a_ptr1, %[lda]\n" "movi v27.4s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q8, [%[b_ptr0]]\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b.ne 3b\n" - "2:\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1125,56 +1028,56 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1193,7 +1096,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1206,526 +1109,473 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "c_ptr1 .req X3\n" "c_ptr2 .req X4\n" "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v18.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v19.4s, #0\n" - "ldr q3, [a_ptr3]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v20.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v21.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v22.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v23.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q14, [%[b_ptr0], #0x60]\n" "movi v24.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "movi v25.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr q1, [a_ptr1]\n" "movi v26.4s, #0\n" - "ldr q14, [%[b_ptr0], #0x60]\n" + "add a_ptr2, a_ptr1, %[lda]\n" "movi v27.4s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr q2, [a_ptr2]\n" "movi v28.4s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" + "add a_ptr3, a_ptr2, %[lda]\n" "movi v29.4s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" + "ldr q3, [a_ptr3]\n" "movi v30.4s, #0\n" - "add a_ptr3, a_ptr3, #0x10\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "movi v31.4s, #0\n" - "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #0x10]\n" - "ldr q18, [%[c_ptr0], #0x20]\n" - "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" - "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" - "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" - "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" - "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" - "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" - "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" - "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" - "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" - "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" - "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" - "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" - "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" - "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" - "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" - "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" - "ldr q3, [a_ptr3]\n" - "ldr q8, [%[b_ptr0]]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "add a_ptr1, a_ptr1, #0x10\n" - "ldr q10, [%[b_ptr0], #0x20]\n" "add a_ptr2, a_ptr2, #0x10\n" - "ldr q11, [%[b_ptr0], #0x30]\n" "add a_ptr3, a_ptr3, #0x10\n" - "ldr q12, [%[b_ptr0], #0x40]\n" - "ldr q13, [%[b_ptr0], #0x50]\n" - "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 2f\n" - "3:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[loops], 1f\n" + "2:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" "ldr q3, [a_ptr3, #-0x10]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b.ne 3b\n" - "2:\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + "b.ne 2b\n" + "1:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" - "cbz %[regs], 4f\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + "cbz %[regs], 3f\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b 5f\n" - "4:\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" + "b 4f\n" + "3:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" - "5:\n" - "cbz %[blocks], 6f\n" - "7:\n" + ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1734,69 +1584,69 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "b.ne 7b\n" - "6:\n" - "cbz %[odds], 8f\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "b.ne 6b\n" + "5:\n" + "cbz %[odds], 7f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "ld1 {v3.b}[0], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "ld1 {v3.b}[1], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" "ld1 {v3.b}[2], [a_ptr3]\n" - "9:\n" + "8:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "8:\n" + ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" + "7:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1821,7 +1671,7 @@ void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_ ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp index 48731efc57..5295650e7b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_hybrid_u8u32_dot_16x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); -void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void a64_hybrid_u8u32_dot_16x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); +void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class hybrid_u8u32_dot_16x4 { @@ -41,7 +41,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -54,11 +54,26 @@ public: return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp index e343d58fa0..e8ed0c311e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -60,7 +62,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(uint32_t); uint32_t *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -78,7 +80,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "temploadreg1 .req X1\n" "temploadreg2 .req X2\n" "temploadreg3 .req X3\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -97,21 +99,16 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v17.4s, v17.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -119,19 +116,19 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "subs %[loops], %[loops], #0x1\n" @@ -146,108 +143,108 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" "ldr d8, [%[b_ptr0]]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" @@ -266,21 +263,21 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" @@ -291,134 +288,134 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" "ins v13.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" "ins v12.d[1], temploadreg0\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -430,10 +427,10 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -449,10 +446,10 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -464,7 +461,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -478,7 +475,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "temploadreg3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -503,31 +500,22 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -535,33 +523,33 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "subs %[loops], %[loops], #0x1\n" @@ -571,83 +559,83 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x20\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v1.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" @@ -656,55 +644,55 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -723,211 +711,211 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr d13, [%[b_ptr0], #0x50]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ins v14.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -939,16 +927,16 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -967,14 +955,14 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -992,7 +980,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; @@ -1010,7 +998,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -1042,41 +1030,28 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -1085,149 +1060,149 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ins v14.d[1], temploadreg2\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" "ins v10.d[1], temploadreg2\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" "ldr d11, [%[b_ptr0], #0x30]\n" "add a_ptr2, a_ptr2, #0x20\n" @@ -1235,65 +1210,65 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" "ldr d2, [a_ptr2, #-0x10]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" "ins v8.d[1], temploadreg0\n" "ins v2.d[1], temploadreg2\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" "ldr d14, [%[b_ptr0], #0x60]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" "ins v9.d[1], temploadreg1\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" @@ -1313,263 +1288,263 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -1581,22 +1556,22 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -1618,18 +1593,18 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -1653,7 +1628,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory" ); break; @@ -1676,7 +1651,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -1714,51 +1689,34 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" @@ -1767,261 +1725,261 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ins v14.d[1], temploadreg2\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr d0, [%[a_ptr0], #-0x10]\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #-0x8]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ins v0.d[1], temploadreg0\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr d1, [a_ptr1, #-0x10]\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr temploadreg1, [a_ptr1, #-0x8]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ins v1.d[1], temploadreg1\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr d2, [a_ptr2, #-0x10]\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr temploadreg2, [a_ptr2, #-0x8]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ins v2.d[1], temploadreg2\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr d3, [a_ptr3, #-0x10]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" "ldr temploadreg3, [a_ptr3, #-0x8]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ins v3.d[1], temploadreg3\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" @@ -2036,315 +1994,315 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "prfm PSTL1KEEP, [c_ptr3]\n" "ins v15.d[1], temploadreg3\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg0, [%[a_ptr0], #0x8]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr d5, [a_ptr1]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr temploadreg1, [a_ptr1, #0x8]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr d6, [a_ptr2]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr temploadreg2, [a_ptr2, #0x8]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr d7, [a_ptr3]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr temploadreg3, [a_ptr3, #0x8]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ins v4.d[1], temploadreg0\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ins v5.d[1], temploadreg1\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ins v6.d[1], temploadreg2\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ins v7.d[1], temploadreg3\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr d8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x78]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "ldr d9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #-0x68]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ldr temploadreg2, [%[b_ptr0], #-0x58]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr temploadreg3, [%[b_ptr0], #-0x48]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr d10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #-0x38]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr d11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #-0x28]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #-0x18]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr d12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr d13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr d14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ins v8.d[1], temploadreg0\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr d9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ins v9.d[1], temploadreg1\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr d10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ins v10.d[1], temploadreg2\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr d11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ins v11.d[1], temploadreg3\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ins v12.d[1], temploadreg0\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr d13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ins v13.d[1], temploadreg1\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr d14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ins v14.d[1], temploadreg2\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr d15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ins v15.d[1], temploadreg3\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -2356,28 +2314,28 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -2402,22 +2360,22 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -2447,7 +2405,7 @@ void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, ".unreq temploadreg2\n" ".unreq temploadreg3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp index 6301fa5657..23d919a64c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,17 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { + UNUSED(bias); + UNUSED(act); + const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -49,7 +52,6 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint for (int x0=0; x0<N; x0+=16ul) { const long width = std::min((unsigned long)N-x0, 16ul); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long blocks = blocks_count; @@ -60,7 +62,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint uint32_t result_buffer[64]; const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(uint32_t); uint32_t *c_ptr_real = c_ptr0; - if (use_result_buffer && !beta0) { + if (use_result_buffer && append) { for(int cy=0; cy<std::min(M-y, 4); cy++) { for(unsigned int cx=0; cx<width; cx++) { result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx]; @@ -74,7 +76,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint switch(M-y) { case 1: __asm __volatile ( - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -92,93 +94,88 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v17.4s, v17.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" @@ -189,93 +186,93 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -287,10 +284,10 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -306,10 +303,10 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -317,7 +314,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "str q19, [%[c_ptr0], #0x30]\n" "add %[c_ptr0], %[c_ptr0], #0x40\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" ); break; @@ -327,7 +324,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "c_ptr1 .req X1\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -351,287 +348,278 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v21.4s, v21.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v22.4s, v22.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "b.ne 3b\n" "2:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -643,16 +631,16 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -671,14 +659,14 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -692,7 +680,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" ); break; @@ -706,7 +694,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add c_ptr1, %[c_ptr0], %[ldc]\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -736,190 +724,177 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v25.4s, v25.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v26.4s, v26.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v27.4s, v27.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "b.ne 3b\n" "2:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" @@ -927,193 +902,193 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -1125,22 +1100,22 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -1162,18 +1137,18 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -1193,7 +1168,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1212,7 +1187,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add c_ptr2, c_ptr1, %[ldc]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -1248,236 +1223,219 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1r {v15.4s}, [%[betaptr]]\n" "ldr q16, [%[c_ptr0]]\n" "ldr q17, [%[c_ptr0], #0x10]\n" "ldr q18, [%[c_ptr0], #0x20]\n" "ldr q19, [%[c_ptr0], #0x30]\n" - "mul v16.4s, v16.4s, v15.4s\n" "ldr q20, [c_ptr1]\n" - "mul v17.4s, v17.4s, v15.4s\n" "ldr q21, [c_ptr1, #0x10]\n" - "mul v18.4s, v18.4s, v15.4s\n" "ldr q22, [c_ptr1, #0x20]\n" - "mul v19.4s, v19.4s, v15.4s\n" "ldr q23, [c_ptr1, #0x30]\n" - "mul v20.4s, v20.4s, v15.4s\n" "ldr q24, [c_ptr2]\n" - "mul v21.4s, v21.4s, v15.4s\n" "ldr q25, [c_ptr2, #0x10]\n" - "mul v22.4s, v22.4s, v15.4s\n" "ldr q26, [c_ptr2, #0x20]\n" - "mul v23.4s, v23.4s, v15.4s\n" "ldr q27, [c_ptr2, #0x30]\n" - "mul v24.4s, v24.4s, v15.4s\n" "ldr q28, [c_ptr3]\n" - "mul v25.4s, v25.4s, v15.4s\n" "ldr q29, [c_ptr3, #0x10]\n" - "mul v26.4s, v26.4s, v15.4s\n" "ldr q30, [c_ptr3, #0x20]\n" - "mul v27.4s, v27.4s, v15.4s\n" "ldr q31, [c_ptr3, #0x30]\n" - "mul v28.4s, v28.4s, v15.4s\n" "ldr q0, [%[a_ptr0]]\n" - "mul v29.4s, v29.4s, v15.4s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q1, [a_ptr1]\n" - "mul v30.4s, v30.4s, v15.4s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ldr q2, [a_ptr2]\n" - "mul v31.4s, v31.4s, v15.4s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ldr q3, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ldr q8, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr q9, [%[b_ptr0], #0x10]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ldr q10, [%[b_ptr0], #0x20]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ldr q11, [%[b_ptr0], #0x30]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ldr q12, [%[b_ptr0], #0x40]\n" "ldr q13, [%[b_ptr0], #0x50]\n" "ldr q14, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "cbz %[loops], 2f\n" "3:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "subs %[loops], %[loops], #0x1\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x20\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x20\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x20\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" "ldr q0, [%[a_ptr0], #-0x10]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" "ldr q1, [a_ptr1, #-0x10]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" "ldr q2, [a_ptr2, #-0x10]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" "ldr q3, [a_ptr3, #-0x10]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "b.ne 3b\n" "2:\n" "ldr q15, [%[b_ptr0], #-0x10]\n" @@ -1486,243 +1444,243 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" "cbz %[regs], 4f\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr q4, [%[a_ptr0]]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "ldr q5, [a_ptr1]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "ldr q6, [a_ptr2]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr q7, [a_ptr3]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" "add a_ptr1, a_ptr1, #0x10\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "add a_ptr2, a_ptr2, #0x10\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" "add a_ptr3, a_ptr3, #0x10\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x100\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" "ldr q8, [%[b_ptr0], #-0x80]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" "ldr q9, [%[b_ptr0], #-0x70]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" "ldr q10, [%[b_ptr0], #-0x60]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" "ldr q11, [%[b_ptr0], #-0x50]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" "ldr q12, [%[b_ptr0], #-0x40]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" "ldr q13, [%[b_ptr0], #-0x30]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" "ldr q14, [%[b_ptr0], #-0x20]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "ldr q15, [%[b_ptr0], #-0x10]\n" - ".word 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" - ".word 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" - ".word 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" - ".word 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" + ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n" + ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n" + ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" - ".word 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" - ".word 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" - ".word 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" + ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n" + ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n" + ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n" + ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" - ".word 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" - ".word 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" - ".word 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" + ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n" + ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n" + ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n" + ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" - ".word 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" - ".word 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" - ".word 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" + ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n" + ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n" + ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n" + ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" - ".word 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" - ".word 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" - ".word 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" + ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n" + ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n" + ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n" + ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" - ".word 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" - ".word 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" - ".word 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" + ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n" + ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n" + ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n" + ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" - ".word 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" - ".word 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" - ".word 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" + ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n" + ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n" + ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n" + ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" - ".word 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" - ".word 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" - ".word 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" + ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n" + ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n" + ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n" + ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" - ".word 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" - ".word 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" - ".word 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" - ".word 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" - ".word 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" - ".word 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" - ".word 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" - ".word 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" - ".word 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" - ".word 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" - ".word 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" - ".word 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" - ".word 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" - ".word 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" - ".word 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" - ".word 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" - ".word 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" - ".word 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" - ".word 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" - ".word 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" - ".word 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" - ".word 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" - ".word 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" - ".word 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" - ".word 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" - ".word 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" - ".word 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" - ".word 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" - ".word 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" - ".word 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" + ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n" + ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n" + ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n" + ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n" + ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n" + ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n" + ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n" + ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n" + ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n" + ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n" + ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n" + ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n" + ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n" + ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n" + ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n" + ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n" + ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n" + ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n" + ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n" + ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n" + ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n" + ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n" + ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n" + ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n" + ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n" + ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n" + ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n" + ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n" + ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n" "b 5f\n" "4:\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" "ldr q8, [%[b_ptr0]]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" "ldr q9, [%[b_ptr0], #0x10]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" "ldr q10, [%[b_ptr0], #0x20]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" - ".word 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" - ".word 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" - ".word 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" + ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n" + ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n" + ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n" + ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n" "ldr q12, [%[b_ptr0], #0x40]\n" - ".word 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" - ".word 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" - ".word 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" - ".word 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" + ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n" + ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n" + ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n" + ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n" "ldr q13, [%[b_ptr0], #0x50]\n" - ".word 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" - ".word 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" - ".word 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" - ".word 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" + ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n" + ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n" + ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n" + ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n" "ldr q14, [%[b_ptr0], #0x60]\n" - ".word 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" - ".word 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" - ".word 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" - ".word 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" + ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n" + ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n" + ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n" + ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n" "ldr q15, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" - ".word 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" - ".word 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" - ".word 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" - ".word 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" - ".word 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" - ".word 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" - ".word 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" - ".word 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" - ".word 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" - ".word 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" - ".word 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" - ".word 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" - ".word 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" - ".word 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" - ".word 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" - ".word 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" - ".word 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" - ".word 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" - ".word 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" - ".word 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" - ".word 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" - ".word 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" - ".word 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" - ".word 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" - ".word 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" - ".word 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" - ".word 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" - ".word 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" - ".word 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" - ".word 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n" + ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n" + ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n" + ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n" + ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n" + ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n" + ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n" + ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n" + ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n" + ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n" + ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n" + ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n" + ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n" + ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n" + ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n" + ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n" + ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n" + ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n" + ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n" + ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n" + ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n" + ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n" + ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n" + ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n" + ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n" + ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n" + ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n" + ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n" + ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n" + ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n" + ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n" "5:\n" "cbz %[blocks], 6f\n" "7:\n" @@ -1734,28 +1692,28 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "add %[a_ptr0], %[a_ptr0], #0x4\n" "ldr q11, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" "ldr s1, [a_ptr1]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" "add a_ptr1, a_ptr1, #0x4\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" "ldr s2, [a_ptr2]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" "add a_ptr2, a_ptr2, #0x4\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" "ldr s3, [a_ptr3]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" "add a_ptr3, a_ptr3, #0x4\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "b.ne 7b\n" "6:\n" "cbz %[odds], 8f\n" @@ -1780,22 +1738,22 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" "ldr q11, [%[b_ptr0], #0x30]\n" - ".word 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" - ".word 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" - ".word 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".word 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" - ".word 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" - ".word 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" - ".word 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" - ".word 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" - ".word 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" - ".word 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" - ".word 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" - ".word 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" - ".word 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" - ".word 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" - ".word 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" - ".word 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" + ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n" + ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n" + ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n" + ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n" + ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n" + ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" + ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n" + ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n" + ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n" + ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n" + ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n" "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" @@ -1821,7 +1779,7 @@ void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp index 06622d6f2e..a86e8ec068 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,51 +23,61 @@ */ #pragma once -#ifdef __ARM_FEATURE_SVE - - +#ifdef __aarch64__ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_fp32_mla_1VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int); +void a64_native_fp32_mla_16x4(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); -class smallK_fp32_mla_1VLx4 +class native_fp32_mla_16x4 { public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } static unsigned int out_width() { - return get_vector_length<float>() * 1; + return 16; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return false; + } - - // Default to the generic kernel - kern_type kernel=sve_smallK_fp32_mla_1VLx4; - - smallK_fp32_mla_1VLx4(const CPUInfo *ci) + static constexpr bool supports_bias() { + return true; + } + static constexpr bool supports_activation() + { + return true; } + + + + // Default to the generic kernel + kern_type kernel=a64_native_fp32_mla_16x4; + + native_fp32_mla_16x4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp new file mode 100644 index 0000000000..82e7333ee3 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp @@ -0,0 +1,1708 @@ +/* + * Copyright (c) 2018-2019 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __aarch64__ + +#include <algorithm> + +#include "arm_gemm.hpp" + +#include "../../asmlib.hpp" +#include "../../utils.hpp" + +namespace arm_gemm { + +void a64_native_fp32_mla_16x4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { + const long loops_count = ((K + 4) / 8) - 1; + K -= loops_count * 8; + const long regs_count = (K / 4) - 1; + K -= (regs_count + 1) * 4; + const long blocks_count = K / 1; + float nullbias[16]; + if (!append && !bias) { + memset(nullbias, 0, (16 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + for (int y=0; y<M; y+=4) { + const float * const a_ptr0_base = A + (y * lda); + const unsigned long ldab = lda * sizeof(float); + + float *c_ptr0 = C + (y * ldc); + + for (int x0=0; x0<N; x0+=16ul) { + const long width = std::min((unsigned long)N-x0, 16ul); + long loops = loops_count; + long regs = regs_count; + long blocks = blocks_count; + const float *a_ptr0 = a_ptr0_base; + const float *b_ptr0 = B + x0; + long ldbb = ldb * sizeof(float); + const unsigned long ldcb = ldc * sizeof(float); + const float *biasptr = bias ? bias+x0 : nullbias; + + switch(M-y) { + case 1: + __asm __volatile ( + "ldr q16, [%[biasptr]]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "ldr q0, [%[a_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr q8, [%[b_ptr0]]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q4, [%[a_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "str q17, [%[c_ptr0], #0x10]\n" + "str q18, [%[c_ptr0], #0x20]\n" + "str q19, [%[c_ptr0], #0x30]\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 2: + __asm __volatile ( + "a_ptr1 .req X0\n" + "c_ptr1 .req X1\n" + "ldr q16, [%[biasptr]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov v20.16b, v16.16b\n" + "ldr q18, [%[biasptr], #0x20]\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v21.16b, v17.16b\n" + "ldr q0, [%[a_ptr0]]\n" + "ldr q1, [a_ptr1]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov v22.16b, v18.16b\n" + "ldr q8, [%[b_ptr0]]\n" + "mov v23.16b, v19.16b\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "add a_ptr1, a_ptr1, #0x20\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q1, [a_ptr1, #-0x10]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr s1, [a_ptr1]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add a_ptr1, a_ptr1, #0x4\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" + "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "str q19, [%[c_ptr0], #0x30]\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + "str q20, [c_ptr1]\n" + "str q21, [c_ptr1, #0x10]\n" + "str q22, [c_ptr1, #0x20]\n" + "str q23, [c_ptr1, #0x30]\n" + ".unreq a_ptr1\n" + ".unreq c_ptr1\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory" + ); + break; + case 3: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "c_ptr1 .req X2\n" + "c_ptr2 .req X3\n" + "ldr q16, [%[biasptr]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "mov v20.16b, v16.16b\n" + "ldr q18, [%[biasptr], #0x20]\n" + "mov v24.16b, v16.16b\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v21.16b, v17.16b\n" + "ldr q0, [%[a_ptr0]]\n" + "mov v25.16b, v17.16b\n" + "ldr q1, [a_ptr1]\n" + "mov v22.16b, v18.16b\n" + "ldr q2, [a_ptr2]\n" + "mov v23.16b, v19.16b\n" + "ldr q8, [%[b_ptr0]]\n" + "mov v26.16b, v18.16b\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "mov v27.16b, v19.16b\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add a_ptr2, a_ptr2, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "add a_ptr1, a_ptr1, #0x20\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "add a_ptr2, a_ptr2, #0x20\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q1, [a_ptr1, #-0x10]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q2, [a_ptr2, #-0x10]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" + "prfm PSTL1KEEP, [c_ptr2]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr s1, [a_ptr1]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add a_ptr1, a_ptr1, #0x4\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr s2, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add a_ptr2, a_ptr2, #0x4\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" + "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" + "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" + "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" + "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "str q22, [c_ptr1, #0x20]\n" + "str q23, [c_ptr1, #0x30]\n" + "str q24, [c_ptr2]\n" + "str q25, [c_ptr2, #0x10]\n" + "str q26, [c_ptr2, #0x20]\n" + "str q27, [c_ptr2, #0x30]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory" + ); + break; + default: + case 4: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "c_ptr1 .req X3\n" + "c_ptr2 .req X4\n" + "c_ptr3 .req X5\n" + "ldr q16, [%[biasptr]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q17, [%[biasptr], #0x10]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "mov v20.16b, v16.16b\n" + "ldr q18, [%[biasptr], #0x20]\n" + "mov v24.16b, v16.16b\n" + "ldr q19, [%[biasptr], #0x30]\n" + "mov v21.16b, v17.16b\n" + "ldr q0, [%[a_ptr0]]\n" + "mov v25.16b, v17.16b\n" + "ldr q1, [a_ptr1]\n" + "mov v22.16b, v18.16b\n" + "ldr q2, [a_ptr2]\n" + "mov v23.16b, v19.16b\n" + "ldr q8, [%[b_ptr0]]\n" + "mov v26.16b, v18.16b\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "mov v27.16b, v19.16b\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "mov v28.16b, v16.16b\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "mov v29.16b, v17.16b\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "mov v30.16b, v18.16b\n" + "ldr q3, [a_ptr3]\n" + "mov v31.16b, v19.16b\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "add a_ptr3, a_ptr3, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr q12, [%[b_ptr0]]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "cbz %[loops], 1f\n" + "2:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "ldr q7, [a_ptr3]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "subs %[loops], %[loops], #0x1\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "add a_ptr1, a_ptr1, #0x20\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "add a_ptr2, a_ptr2, #0x20\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "add a_ptr3, a_ptr3, #0x20\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "prfm PLDL1KEEP, [a_ptr2, #0x40]\n" + "fmla v28.4s, v12.4s, v3.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "fmla v29.4s, v13.4s, v3.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "fmla v30.4s, v14.4s, v3.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "fmla v31.4s, v15.4s, v3.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v28.4s, v8.4s, v3.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v29.4s, v9.4s, v3.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v30.4s, v10.4s, v3.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v31.4s, v11.4s, v3.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v28.4s, v12.4s, v3.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v29.4s, v13.4s, v3.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v30.4s, v14.4s, v3.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "ldr q0, [%[a_ptr0], #-0x10]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "ldr q1, [a_ptr1, #-0x10]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "ldr q2, [a_ptr2, #-0x10]\n" + "fmla v31.4s, v15.4s, v3.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "ldr q3, [a_ptr3, #-0x10]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "fmla v28.4s, v8.4s, v7.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "fmla v29.4s, v9.4s, v7.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "fmla v30.4s, v10.4s, v7.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "fmla v31.4s, v11.4s, v7.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "fmla v28.4s, v12.4s, v7.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "fmla v29.4s, v13.4s, v7.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "fmla v30.4s, v14.4s, v7.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "fmla v31.4s, v15.4s, v7.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "fmla v28.4s, v8.4s, v7.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "fmla v29.4s, v9.4s, v7.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "fmla v30.4s, v10.4s, v7.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "fmla v31.4s, v11.4s, v7.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "fmla v28.4s, v12.4s, v7.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "fmla v29.4s, v13.4s, v7.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "fmla v30.4s, v14.4s, v7.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "fmla v31.4s, v15.4s, v7.s[3]\n" + "b.ne 2b\n" + "1:\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "prfm PSTL1KEEP, [%[c_ptr0]]\n" + "prfm PSTL1KEEP, [c_ptr1]\n" + "prfm PSTL1KEEP, [c_ptr2]\n" + "prfm PSTL1KEEP, [c_ptr3]\n" + "cbz %[regs], 3f\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q4, [%[a_ptr0]]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr q5, [a_ptr1]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "ldr q6, [a_ptr2]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "ldr q7, [a_ptr3]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "add a_ptr3, a_ptr3, #0x10\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "fmla v28.4s, v12.4s, v3.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "fmla v29.4s, v13.4s, v3.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "fmla v30.4s, v14.4s, v3.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "fmla v31.4s, v15.4s, v3.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v28.4s, v8.4s, v3.s[2]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v29.4s, v9.4s, v3.s[2]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v30.4s, v10.4s, v3.s[2]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v31.4s, v11.4s, v3.s[2]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v28.4s, v12.4s, v3.s[3]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v29.4s, v13.4s, v3.s[3]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v30.4s, v14.4s, v3.s[3]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "fmla v31.4s, v15.4s, v3.s[3]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v5.s[0]\n" + "fmla v24.4s, v8.4s, v6.s[0]\n" + "fmla v28.4s, v8.4s, v7.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v4.s[0]\n" + "fmla v21.4s, v9.4s, v5.s[0]\n" + "fmla v25.4s, v9.4s, v6.s[0]\n" + "fmla v29.4s, v9.4s, v7.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v4.s[0]\n" + "fmla v22.4s, v10.4s, v5.s[0]\n" + "fmla v26.4s, v10.4s, v6.s[0]\n" + "fmla v30.4s, v10.4s, v7.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v4.s[0]\n" + "fmla v23.4s, v11.4s, v5.s[0]\n" + "fmla v27.4s, v11.4s, v6.s[0]\n" + "fmla v31.4s, v11.4s, v7.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v5.s[1]\n" + "fmla v24.4s, v12.4s, v6.s[1]\n" + "fmla v28.4s, v12.4s, v7.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v4.s[1]\n" + "fmla v21.4s, v13.4s, v5.s[1]\n" + "fmla v25.4s, v13.4s, v6.s[1]\n" + "fmla v29.4s, v13.4s, v7.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v4.s[1]\n" + "fmla v22.4s, v14.4s, v5.s[1]\n" + "fmla v26.4s, v14.4s, v6.s[1]\n" + "fmla v30.4s, v14.4s, v7.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v4.s[1]\n" + "fmla v23.4s, v15.4s, v5.s[1]\n" + "fmla v27.4s, v15.4s, v6.s[1]\n" + "fmla v31.4s, v15.4s, v7.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v4.s[2]\n" + "fmla v20.4s, v8.4s, v5.s[2]\n" + "fmla v24.4s, v8.4s, v6.s[2]\n" + "fmla v28.4s, v8.4s, v7.s[2]\n" + "fmla v17.4s, v9.4s, v4.s[2]\n" + "fmla v21.4s, v9.4s, v5.s[2]\n" + "fmla v25.4s, v9.4s, v6.s[2]\n" + "fmla v29.4s, v9.4s, v7.s[2]\n" + "fmla v18.4s, v10.4s, v4.s[2]\n" + "fmla v22.4s, v10.4s, v5.s[2]\n" + "fmla v26.4s, v10.4s, v6.s[2]\n" + "fmla v30.4s, v10.4s, v7.s[2]\n" + "fmla v19.4s, v11.4s, v4.s[2]\n" + "fmla v23.4s, v11.4s, v5.s[2]\n" + "fmla v27.4s, v11.4s, v6.s[2]\n" + "fmla v31.4s, v11.4s, v7.s[2]\n" + "fmla v16.4s, v12.4s, v4.s[3]\n" + "fmla v20.4s, v12.4s, v5.s[3]\n" + "fmla v24.4s, v12.4s, v6.s[3]\n" + "fmla v28.4s, v12.4s, v7.s[3]\n" + "fmla v17.4s, v13.4s, v4.s[3]\n" + "fmla v21.4s, v13.4s, v5.s[3]\n" + "fmla v25.4s, v13.4s, v6.s[3]\n" + "fmla v29.4s, v13.4s, v7.s[3]\n" + "fmla v18.4s, v14.4s, v4.s[3]\n" + "fmla v22.4s, v14.4s, v5.s[3]\n" + "fmla v26.4s, v14.4s, v6.s[3]\n" + "fmla v30.4s, v14.4s, v7.s[3]\n" + "fmla v19.4s, v15.4s, v4.s[3]\n" + "fmla v23.4s, v15.4s, v5.s[3]\n" + "fmla v27.4s, v15.4s, v6.s[3]\n" + "fmla v31.4s, v15.4s, v7.s[3]\n" + "b 4f\n" + "3:\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "ldr q8, [%[b_ptr0]]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v12.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla v20.4s, v12.4s, v1.s[1]\n" + "fmla v24.4s, v12.4s, v2.s[1]\n" + "fmla v28.4s, v12.4s, v3.s[1]\n" + "ldr q12, [%[b_ptr0]]\n" + "fmla v17.4s, v13.4s, v0.s[1]\n" + "fmla v21.4s, v13.4s, v1.s[1]\n" + "fmla v25.4s, v13.4s, v2.s[1]\n" + "fmla v29.4s, v13.4s, v3.s[1]\n" + "ldr q13, [%[b_ptr0], #0x10]\n" + "fmla v18.4s, v14.4s, v0.s[1]\n" + "fmla v22.4s, v14.4s, v1.s[1]\n" + "fmla v26.4s, v14.4s, v2.s[1]\n" + "fmla v30.4s, v14.4s, v3.s[1]\n" + "ldr q14, [%[b_ptr0], #0x20]\n" + "fmla v19.4s, v15.4s, v0.s[1]\n" + "fmla v23.4s, v15.4s, v1.s[1]\n" + "fmla v27.4s, v15.4s, v2.s[1]\n" + "fmla v31.4s, v15.4s, v3.s[1]\n" + "ldr q15, [%[b_ptr0], #0x30]\n" + "fmla v16.4s, v8.4s, v0.s[2]\n" + "fmla v20.4s, v8.4s, v1.s[2]\n" + "fmla v24.4s, v8.4s, v2.s[2]\n" + "fmla v28.4s, v8.4s, v3.s[2]\n" + "fmla v17.4s, v9.4s, v0.s[2]\n" + "fmla v21.4s, v9.4s, v1.s[2]\n" + "fmla v25.4s, v9.4s, v2.s[2]\n" + "fmla v29.4s, v9.4s, v3.s[2]\n" + "fmla v18.4s, v10.4s, v0.s[2]\n" + "fmla v22.4s, v10.4s, v1.s[2]\n" + "fmla v26.4s, v10.4s, v2.s[2]\n" + "fmla v30.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v11.4s, v0.s[2]\n" + "fmla v23.4s, v11.4s, v1.s[2]\n" + "fmla v27.4s, v11.4s, v2.s[2]\n" + "fmla v31.4s, v11.4s, v3.s[2]\n" + "fmla v16.4s, v12.4s, v0.s[3]\n" + "fmla v20.4s, v12.4s, v1.s[3]\n" + "fmla v24.4s, v12.4s, v2.s[3]\n" + "fmla v28.4s, v12.4s, v3.s[3]\n" + "fmla v17.4s, v13.4s, v0.s[3]\n" + "fmla v21.4s, v13.4s, v1.s[3]\n" + "fmla v25.4s, v13.4s, v2.s[3]\n" + "fmla v29.4s, v13.4s, v3.s[3]\n" + "fmla v18.4s, v14.4s, v0.s[3]\n" + "fmla v22.4s, v14.4s, v1.s[3]\n" + "fmla v26.4s, v14.4s, v2.s[3]\n" + "fmla v30.4s, v14.4s, v3.s[3]\n" + "fmla v19.4s, v15.4s, v0.s[3]\n" + "fmla v23.4s, v15.4s, v1.s[3]\n" + "fmla v27.4s, v15.4s, v2.s[3]\n" + "fmla v31.4s, v15.4s, v3.s[3]\n" + "4:\n" + "cbz %[blocks], 5f\n" + "6:\n" + "subs %[blocks], %[blocks], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr q8, [%[b_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x4\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr s1, [a_ptr1]\n" + "fmla v16.4s, v8.4s, v0.s[0]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "add a_ptr1, a_ptr1, #0x4\n" + "fmla v20.4s, v8.4s, v1.s[0]\n" + "ldr s2, [a_ptr2]\n" + "fmla v17.4s, v9.4s, v0.s[0]\n" + "add a_ptr2, a_ptr2, #0x4\n" + "fmla v21.4s, v9.4s, v1.s[0]\n" + "ldr s3, [a_ptr3]\n" + "fmla v24.4s, v8.4s, v2.s[0]\n" + "add a_ptr3, a_ptr3, #0x4\n" + "fmla v25.4s, v9.4s, v2.s[0]\n" + "fmla v28.4s, v8.4s, v3.s[0]\n" + "fmla v29.4s, v9.4s, v3.s[0]\n" + "fmla v18.4s, v10.4s, v0.s[0]\n" + "fmla v22.4s, v10.4s, v1.s[0]\n" + "fmla v26.4s, v10.4s, v2.s[0]\n" + "fmla v30.4s, v10.4s, v3.s[0]\n" + "fmla v19.4s, v11.4s, v0.s[0]\n" + "fmla v23.4s, v11.4s, v1.s[0]\n" + "fmla v27.4s, v11.4s, v2.s[0]\n" + "fmla v31.4s, v11.4s, v3.s[0]\n" + "b.ne 6b\n" + "5:\n" + "ld1r {v14.4s}, [%[minptr]]\n" + "ld1r {v15.4s}, [%[maxptr]]\n" + "fmax v16.4s, v16.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v14.4s\n" + "fmax v18.4s, v18.4s, v14.4s\n" + "fmax v19.4s, v19.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q16, [%[c_ptr0]]\n" + "fmax v20.4s, v20.4s, v14.4s\n" + "fmax v21.4s, v21.4s, v14.4s\n" + "fmax v22.4s, v22.4s, v14.4s\n" + "str q17, [%[c_ptr0], #0x10]\n" + "fmax v23.4s, v23.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "str q18, [%[c_ptr0], #0x20]\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmax v24.4s, v24.4s, v14.4s\n" + "str q19, [%[c_ptr0], #0x30]\n" + "fmax v25.4s, v25.4s, v14.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x40\n" + "fmax v26.4s, v26.4s, v14.4s\n" + "str q20, [c_ptr1]\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v14.4s\n" + "str q21, [c_ptr1, #0x10]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v14.4s\n" + "fmax v29.4s, v29.4s, v14.4s\n" + "str q22, [c_ptr1, #0x20]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "str q23, [c_ptr1, #0x30]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "str q24, [c_ptr2]\n" + "fmin v31.4s, v31.4s, v15.4s\n" + "str q25, [c_ptr2, #0x10]\n" + "str q26, [c_ptr2, #0x20]\n" + "str q27, [c_ptr2, #0x30]\n" + "str q28, [c_ptr3]\n" + "str q29, [c_ptr3, #0x10]\n" + "str q30, [c_ptr3, #0x20]\n" + "str q31, [c_ptr3, #0x30]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [ldb] "r" (ldbb) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" + ); + break; + } + + } + } +} + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp deleted file mode 100644 index 9b70827125..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __aarch64__ - -namespace arm_gemm { - -// Actual kernel implementations -void a64_sgemm_nativeA_pretransposeB_16x4(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int); - -// Native A/Pretranspose B SGEMM "strategy" class. -// -// This describes the characteristics of a family of kernels, in terms of -// the required interleave properties and the output block size. -// -// All kernels in the family must share these characteristics. The actual -// kernel to be used can be chosen at runtime, based on the CPUInfo -// structure. -class sgemm_nativeA_pretransposeB_16x4 { -public: - typedef float operand_type; - typedef float result_type; - - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, unsigned int, unsigned int, unsigned int); - - /* Desired data layout for B buffer (used for pretranspose) */ - static const int B_interleave = 16; - static const int B_block = 1; - static const bool B_transpose = true; - - /* Kernel blocking parameters */ - static unsigned int out_width() { - return 16; - } - - static unsigned int out_height() { - return 4; - } - - static unsigned int k_unroll() { - return 1; - } - - StdTransformsFixed<operand_type, result_type, 4, 16> transforms = {}; - - // Default to the generic kernel - kern_type kernel=a64_sgemm_nativeA_pretransposeB_16x4; - - sgemm_nativeA_pretransposeB_16x4(const CPUInfo *ci) { - UNUSED(ci); - } -}; - -} // namespace arm_gemm - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp deleted file mode 100644 index c51c8ff863..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_nativeA_pretransposeB_16x4/generic.cpp +++ /dev/null @@ -1,970 +0,0 @@ -/* - * Copyright (c) 2019 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __aarch64__ - -#include "../../asmlib.hpp" -#include "../../utils.hpp" - -#include <algorithm> -#include <cstddef> -#include <cstring> - -#include <arm_neon.h> - -namespace arm_gemm { - -void a64_sgemm_nativeA_pretransposeB_16x4(const float *A, int lda, const float *B_panel, float *C, int ldc, float beta, unsigned int numrows, unsigned int numcols, unsigned int K) { - const bool oddk = ((K % 8) >= 4); - const bool beta0 = (beta == 0.0f); - const unsigned int oddones = (K % 4); - - /* Use some small temporary arrays to cope with "ragged" M/N sizes. - * - * "dummy_A_buf" is used to avoid overreading the A input for ragged M, - * and also for output if N is not ragged. - * - * Since the B input is pretransposed it will be padded as needed, so no - * need to worry about overreading that. - * - * "C_buf" is used to avoid overreading or overwriting the output for - * ragged N cases. - */ - float dummy_A_buf[16]; - float C_buf[64]; - - std::memset(dummy_A_buf, 0, sizeof(dummy_A_buf)); - std::memset(C_buf, 0, sizeof(C_buf)); - - for (unsigned int y=0; y<numrows; y+=4) { - const float *b_ptr = B_panel; - const unsigned int active_rows = std::min(numrows - y, 4U); - - /* Increment values to be used to advance A pointers - these get set - * to zero when the corresponding row isn't being used due to ragged - * M, so it will just read the dummy buffer repeatedly. Values are - * in bytes (8x sizeof(float)). */ - const unsigned long a_incr1 = (active_rows > 1) ? 32 : 0; - const unsigned long a_incr2 = (active_rows > 2) ? 32 : 0; - const unsigned long a_incr3 = (active_rows > 3) ? 32 : 0; - - /* Starting points for A pointers on this loop */ - const float * const a_ptr0_base = A + (y * lda); - const float * const a_ptr1_base = (active_rows > 1) ? (a_ptr0_base + lda) : dummy_A_buf; - const float * const a_ptr2_base = (active_rows > 2) ? (a_ptr1_base + lda) : dummy_A_buf; - const float * const a_ptr3_base = (active_rows > 3) ? (a_ptr2_base + lda) : dummy_A_buf; - - /* Starting points for C pointers on this loop */ - float *c_ptr0 = C + (y * ldc); - float *c_ptr1 = (active_rows > 1) ? (c_ptr0 + ldc) : dummy_A_buf; - float *c_ptr2 = (active_rows > 2) ? (c_ptr1 + ldc) : dummy_A_buf; - float *c_ptr3 = (active_rows > 3) ? (c_ptr2 + ldc) : dummy_A_buf; - - for (unsigned int x0=0; x0<numcols; x0+=16) { - const unsigned int active_cols = std::min(numcols - x0, 16U); - const bool use_result_buf = (active_cols < 16); - - /* Reset the A pointers for this loop. */ - const float *a_ptr0 = a_ptr0_base; - const float *a_ptr1 = a_ptr1_base; - const float *a_ptr2 = a_ptr2_base; - const float *a_ptr3 = a_ptr3_base; - - /* Override C pointers if the result buffer is in use. */ - if (use_result_buf) { - c_ptr0 = C_buf; - c_ptr1 = C_buf + 16; - c_ptr2 = C_buf + 32; - c_ptr3 = C_buf + 48; - - /* If beta is non-zero, prepopulate the result buffer */ - if (!beta0) { - for (unsigned int row=0; row<active_rows; row++) { - for (unsigned int col=0; col<active_cols; col++) { - C_buf[row * 16 + col] = C[((y + row) * ldc) + (x0 + col)]; - } - } - } - } - - unsigned int loops = ((K+4)/8) - 1; - unsigned int odds = oddones; - - __asm __volatile ( - "a0 .req v0\n" - "a1 .req v1\n" - "a2 .req v2\n" - "a3 .req v3\n" - "a0a .req v4\n" - "a1a .req v5\n" - "a2a .req v6\n" - "a3a .req v7\n" - "bb0 .req v8\n" - "bb1 .req v9\n" - "bb2 .req v10\n" - "bb3 .req v11\n" - "b0a .req v12\n" - "b1a .req v13\n" - "b2a .req v14\n" - "b3a .req v15\n" - - "a0q .req q0\n" - "a1q .req q1\n" - "a2q .req q2\n" - "a3q .req q3\n" - "a0aq .req q4\n" - "a1aq .req q5\n" - "a2aq .req q6\n" - "a3aq .req q7\n" - "b0q .req q8\n" - "b1q .req q9\n" - "b2q .req q10\n" - "b3q .req q11\n" - "b0aq .req q12\n" - "b1aq .req q13\n" - "b2aq .req q14\n" - "b3aq .req q15\n" - - "movi v16.4s, #0x0\n" - "ldr a0q, [%[a_ptr0]]\n" - "movi v17.4s, #0x0\n" - "ldr b0q, [%[b_ptr]]\n" - "movi v18.4s, #0x0\n" - "ldr b1q, [%[b_ptr], #16]\n" - "movi v19.4s, #0x0\n" - "ldr b2q, [%[b_ptr], #32]\n" - "movi v20.4s, #0x0\n" - "ldr b3q, [%[b_ptr], #48]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "movi v21.4s, #0x0\n" - "ldr a1q, [%[a_ptr1]]\n" - "movi v22.4s, #0x0\n" - "ldr a2q, [%[a_ptr2]]\n" - "movi v23.4s, #0x0\n" - "ldr a3q, [%[a_ptr3]]\n" - "movi v24.4s, #0x0\n" - "ldr b0aq, [%[b_ptr]]\n" - "movi v25.4s, #0x0\n" - "ldr b1aq, [%[b_ptr], #16]\n" - "movi v26.4s, #0x0\n" - "ldr b2aq, [%[b_ptr], #32]\n" - "cbz %w[beta0], 5f\n" - "movi v27.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0x40]") - "movi v28.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0x80]") - "movi v29.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0xC0]") - "movi v30.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0x100]") - "movi v31.4s, #0x0\n" - ASM_PREFETCH("[%[b_ptr], #0x140]") - ASM_PREFETCH("[%[b_ptr], #0x180]") - ASM_PREFETCH("[%[b_ptr], #0x1C0]") - ASM_PREFETCH("[%[b_ptr], #0x200]") - - // Skip if no complete loops. - "cbz %w[loops], 4f\n" - "b 1f\n" - - // If beta is non-zero, need to load and multiply by beta - "5:\n" - "ld1r {v4.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #16]\n" - "ldr q18, [%[c_ptr0], #32]\n" - "ldr q19, [%[c_ptr0], #48]\n" - - "ldr q20, [%[c_ptr1]]\n" - "fmul v16.4s, v16.4s, v4.4s\n" - "ldr q21, [%[c_ptr1], #16]\n" - "fmul v17.4s, v17.4s, v4.4s\n" - "ldr q22, [%[c_ptr1], #32]\n" - "fmul v18.4s, v18.4s, v4.4s\n" - "ldr q23, [%[c_ptr1], #48]\n" - "fmul v19.4s, v19.4s, v4.4s\n" - - "ldr q24, [%[c_ptr2]]\n" - "fmul v20.4s, v20.4s, v4.4s\n" - "ldr q25, [%[c_ptr2], #16]\n" - "fmul v21.4s, v21.4s, v4.4s\n" - "ldr q26, [%[c_ptr2], #32]\n" - "fmul v22.4s, v22.4s, v4.4s\n" - "ldr q27, [%[c_ptr2], #48]\n" - "fmul v23.4s, v23.4s, v4.4s\n" - - "ldr q28, [%[c_ptr3]]\n" - "fmul v24.4s, v24.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x40]") - "ldr q29, [%[c_ptr3], #16]\n" - "fmul v25.4s, v25.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x80]") - "ldr q30, [%[c_ptr3], #32]\n" - "fmul v26.4s, v26.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0xC0]") - "ldr q31, [%[c_ptr3], #48]\n" - "fmul v27.4s, v27.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x100]") - - "fmul v28.4s, v28.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x140]") - "fmul v29.4s, v29.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x180]") - "fmul v30.4s, v30.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x1C0]") - "fmul v31.4s, v31.4s, v4.4s\n" - ASM_PREFETCH("[%[b_ptr], #0x200]") - - "cbz %w[loops], 4f\n" - - "1:\n" - // Unroll 0 - "fmla v16.4s, bb0.4s, a0.s[0]\n" - ASM_PREFETCH("[%[b_ptr], #0x240]") - "fmla v20.4s, bb0.4s, a1.s[0]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - "fmla v24.4s, bb0.4s, a2.s[0]\n" - "fmla v28.4s, bb0.4s, a3.s[0]\n" - "ldr b0q, [%[b_ptr], #64]\n" - - "fmla v17.4s, bb1.4s, a0.s[0]\n" - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "ldr a0aq, [%[a_ptr0], #16]\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #80]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "ldr a1aq, [%[a_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #96]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "ldr a2aq, [%[a_ptr2], #16]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #112]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - ASM_PREFETCH("[%[b_ptr], #0x280]") - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "ldr a3aq, [%[a_ptr3], #16]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr], #128]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "subs %w[loops], %w[loops], #1\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #144]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #160]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #176]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - ASM_PREFETCH("[%[b_ptr], #0x2C0]") - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - "ldr b0q, [%[b_ptr], #192]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "add %[a_ptr0], %[a_ptr0], #32\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - "ldr b1q, [%[b_ptr], #208]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - ASM_PREFETCH("[%[a_ptr0], #0x40]") - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - "ldr b2q, [%[b_ptr], #224]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - ASM_PREFETCH("[%[a_ptr1], #0x40]") - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - "ldr b3q, [%[b_ptr], #240]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "add %[b_ptr], %[b_ptr], #512\n" - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ldr b0aq, [%[b_ptr], #-256]\n" - - "fmla v17.4s, b1a.4s, a0.s[3]\n" - ASM_PREFETCH("[%[b_ptr], #0x100]") - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "ldr b1aq, [%[b_ptr], #-240]\n" - - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - ASM_PREFETCH("[%[a_ptr2], #0x40]") - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ldr b2aq, [%[b_ptr], #-224]\n" - - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "ldr a0q, [%[a_ptr0]]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "ldr b3aq, [%[b_ptr], #-208]\n" - - // Unroll 4 - "fmla v16.4s, bb0.4s, a0a.s[0]\n" - "fmla v20.4s, bb0.4s, a1a.s[0]\n" - ASM_PREFETCH("[%[b_ptr], #0x140]") - "fmla v24.4s, bb0.4s, a2a.s[0]\n" - "fmla v28.4s, bb0.4s, a3a.s[0]\n" - "ldr b0q, [%[b_ptr], #-192]\n" - - "fmla v17.4s, bb1.4s, a0a.s[0]\n" - "fmla v21.4s, bb1.4s, a1a.s[0]\n" - "ldr a1q, [%[a_ptr1]]\n" - "fmla v25.4s, bb1.4s, a2a.s[0]\n" - "fmla v29.4s, bb1.4s, a3a.s[0]\n" - "ldr b1q, [%[b_ptr], #-176]\n" - - "fmla v18.4s, bb2.4s, a0a.s[0]\n" - "fmla v22.4s, bb2.4s, a1a.s[0]\n" - "ldr a2q, [%[a_ptr2]]\n" - "fmla v26.4s, bb2.4s, a2a.s[0]\n" - "fmla v30.4s, bb2.4s, a3a.s[0]\n" - "ldr b2q, [%[b_ptr], #-160]\n" - - "fmla v19.4s, bb3.4s, a0a.s[0]\n" - "fmla v23.4s, bb3.4s, a1a.s[0]\n" - "ldr a3q, [%[a_ptr3]]\n" - "fmla v27.4s, bb3.4s, a2a.s[0]\n" - "fmla v31.4s, bb3.4s, a3a.s[0]\n" - "ldr b3q, [%[b_ptr], #-144]\n" - - // Unroll 5 - "fmla v16.4s, b0a.4s, a0a.s[1]\n" - "fmla v20.4s, b0a.4s, a1a.s[1]\n" - ASM_PREFETCH("[%[b_ptr], #0x180]") - "fmla v24.4s, b0a.4s, a2a.s[1]\n" - "fmla v28.4s, b0a.4s, a3a.s[1]\n" - "ldr b0aq, [%[b_ptr], #-128]\n" - - "fmla v17.4s, b1a.4s, a0a.s[1]\n" - "fmla v21.4s, b1a.4s, a1a.s[1]\n" - ASM_PREFETCH("[%[a_ptr3], #0x40]") - "fmla v25.4s, b1a.4s, a2a.s[1]\n" - "fmla v29.4s, b1a.4s, a3a.s[1]\n" - "ldr b1aq, [%[b_ptr], #-112]\n" - - "fmla v18.4s, b2a.4s, a0a.s[1]\n" - "fmla v22.4s, b2a.4s, a1a.s[1]\n" - "fmla v26.4s, b2a.4s, a2a.s[1]\n" - "fmla v30.4s, b2a.4s, a3a.s[1]\n" - "ldr b2aq, [%[b_ptr], #-96]\n" - - "fmla v19.4s, b3a.4s, a0a.s[1]\n" - "fmla v23.4s, b3a.4s, a1a.s[1]\n" - "fmla v27.4s, b3a.4s, a2a.s[1]\n" - "fmla v31.4s, b3a.4s, a3a.s[1]\n" - "ldr b3aq, [%[b_ptr], #-80]\n" - - // Unroll 6 - "fmla v16.4s, bb0.4s, a0a.s[2]\n" - "fmla v20.4s, bb0.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[b_ptr], #0x1C0]") - "fmla v24.4s, bb0.4s, a2a.s[2]\n" - "fmla v28.4s, bb0.4s, a3a.s[2]\n" - "ldr b0q, [%[b_ptr], #-64]\n" - - "fmla v17.4s, bb1.4s, a0a.s[2]\n" - "fmla v21.4s, bb1.4s, a1a.s[2]\n" - "fmla v25.4s, bb1.4s, a2a.s[2]\n" - "fmla v29.4s, bb1.4s, a3a.s[2]\n" - "ldr b1q, [%[b_ptr], #-48]\n" - - "fmla v18.4s, bb2.4s, a0a.s[2]\n" - "fmla v22.4s, bb2.4s, a1a.s[2]\n" - "fmla v26.4s, bb2.4s, a2a.s[2]\n" - "fmla v30.4s, bb2.4s, a3a.s[2]\n" - "ldr b2q, [%[b_ptr], #-32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[2]\n" - "fmla v23.4s, bb3.4s, a1a.s[2]\n" - "fmla v27.4s, bb3.4s, a2a.s[2]\n" - "fmla v31.4s, bb3.4s, a3a.s[2]\n" - "ldr b3q, [%[b_ptr], #-16]\n" - - // Unroll 7 - "fmla v16.4s, b0a.4s, a0a.s[3]\n" - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[3]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - ASM_PREFETCH("[%[b_ptr], #0x200]") - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[3]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[3]\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - "bne 1b\n" - - // Skip to here - "4:\n" - - // Detached final iteration - // Unroll 0 - "fmla v16.4s, bb0.4s, a0.s[0]\n" - "fmla v20.4s, bb0.4s, a1.s[0]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - "fmla v24.4s, bb0.4s, a2.s[0]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v28.4s, bb0.4s, a3.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[0]\n" - "cbnz %w[oddk], 2f\n" // Deal with odd K before we load a0a - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "ldr a0aq, [%[a_ptr0], #16]\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "ldr a1aq, [%[a_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "ldr a2aq, [%[a_ptr2], #16]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "ldr a3aq, [%[a_ptr3], #16]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "add %[a_ptr0], %[a_ptr0], #32\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 4 - "fmla v16.4s, bb0.4s, a0a.s[0]\n" - "fmla v20.4s, bb0.4s, a1a.s[0]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v24.4s, bb0.4s, a2a.s[0]\n" - "fmla v28.4s, bb0.4s, a3a.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0a.s[0]\n" - "fmla v21.4s, bb1.4s, a1a.s[0]\n" - "fmla v25.4s, bb1.4s, a2a.s[0]\n" - "fmla v29.4s, bb1.4s, a3a.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0a.s[0]\n" - "fmla v22.4s, bb2.4s, a1a.s[0]\n" - "fmla v26.4s, bb2.4s, a2a.s[0]\n" - "fmla v30.4s, bb2.4s, a3a.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[0]\n" - "fmla v23.4s, bb3.4s, a1a.s[0]\n" - "fmla v27.4s, bb3.4s, a2a.s[0]\n" - "fmla v31.4s, bb3.4s, a3a.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 5 - "fmla v16.4s, b0a.4s, a0a.s[1]\n" - "fmla v20.4s, b0a.4s, a1a.s[1]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v24.4s, b0a.4s, a2a.s[1]\n" - "fmla v28.4s, b0a.4s, a3a.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[1]\n" - "fmla v21.4s, b1a.4s, a1a.s[1]\n" - "fmla v25.4s, b1a.4s, a2a.s[1]\n" - "fmla v29.4s, b1a.4s, a3a.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[1]\n" - "fmla v22.4s, b2a.4s, a1a.s[1]\n" - "fmla v26.4s, b2a.4s, a2a.s[1]\n" - "fmla v30.4s, b2a.4s, a3a.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[1]\n" - "fmla v23.4s, b3a.4s, a1a.s[1]\n" - "fmla v27.4s, b3a.4s, a2a.s[1]\n" - "fmla v31.4s, b3a.4s, a3a.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 6 - "fmla v16.4s, bb0.4s, a0a.s[2]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v20.4s, bb0.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[c_ptr0], #0x40]") - "fmla v24.4s, bb0.4s, a2a.s[2]\n" - "fmla v28.4s, bb0.4s, a3a.s[2]\n" - - "fmla v17.4s, bb1.4s, a0a.s[2]\n" - "fmla v21.4s, bb1.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[c_ptr1], #0x40]") - "fmla v25.4s, bb1.4s, a2a.s[2]\n" - "fmla v29.4s, bb1.4s, a3a.s[2]\n" - - "fmla v18.4s, bb2.4s, a0a.s[2]\n" - "fmla v22.4s, bb2.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[c_ptr2], #0x40]") - "fmla v26.4s, bb2.4s, a2a.s[2]\n" - "fmla v30.4s, bb2.4s, a3a.s[2]\n" - - "fmla v19.4s, bb3.4s, a0a.s[2]\n" - "fmla v23.4s, bb3.4s, a1a.s[2]\n" - ASM_PREFETCH("[%[c_ptr3], #0x40]") - "fmla v27.4s, bb3.4s, a2a.s[2]\n" - "fmla v31.4s, bb3.4s, a3a.s[2]\n" - - // Unroll 7 - "fmla v16.4s, b0a.4s, a0a.s[3]\n" - "fmla v17.4s, b1a.4s, a0a.s[3]\n" - "fmla v18.4s, b2a.4s, a0a.s[3]\n" - "fmla v19.4s, b3a.4s, a0a.s[3]\n" - "cbnz %w[odds], 6f\n" - - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - "str q27, [%[c_ptr2], #48]\n" - "b 3f\n" - - // Odd K case: Just do 4 more. - "2:\n" - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #16\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "add %[a_ptr1], %[a_ptr1], #16\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "add %[a_ptr2], %[a_ptr2], #16\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "add %[a_ptr3], %[a_ptr3], #16\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - ASM_PREFETCH("[%[c_ptr0], #0x40]") - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - ASM_PREFETCH("[%[c_ptr1], #0x40]") - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - ASM_PREFETCH("[%[c_ptr2], #0x40]") - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - ASM_PREFETCH("[%[c_ptr3], #0x40]") - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "cbnz %w[odds], 7f\n" - - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "str q27, [%[c_ptr2], #48]\n" - "b 3f\n" - - // "Odd ones" - lead in from even - "6:\n" - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - - "fmla v16.4s, bb0.4s, a0.4s\n" - "beq 9f\n" - "b 8f\n" - - // "Odd ones" - lead in from odd - "7:\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - - "fmla v16.4s, bb0.4s, a0.4s\n" - "beq 9f\n" - - // "Odd ones" - loop - "8:\n" - "fmla v17.4s, bb1.4s, a0.4s\n" - "ld1r {a2.4s}, [%[a_ptr2]], #4\n" - "fmla v18.4s, bb2.4s, a0.4s\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v19.4s, bb3.4s, a0.4s\n" - "ld1r {a3.4s}, [%[a_ptr3]], #4\n" - - "fmla v20.4s, bb0.4s, a1.4s\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v21.4s, bb1.4s, a1.4s\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v22.4s, bb2.4s, a1.4s\n" - "fmla v23.4s, bb3.4s, a1.4s\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - - "fmla v24.4s, bb0.4s, a2.4s\n" - "fmla v28.4s, bb0.4s, a3.4s\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v25.4s, bb1.4s, a2.4s\n" - "fmla v29.4s, bb1.4s, a3.4s\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v26.4s, bb2.4s, a2.4s\n" - "fmla v30.4s, bb2.4s, a3.4s\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v27.4s, bb3.4s, a2.4s\n" - "fmla v31.4s, bb3.4s, a3.4s\n" - "ldr b3q, [%[b_ptr], #48]\n" - "fmla v16.4s, bb0.4s, a0.4s\n" - "bne 8b\n" - - // "Odd ones" - detached final iteration - "9:\n" - "fmla v17.4s, bb1.4s, a0.4s\n" - "ld1r {a2.4s}, [%[a_ptr2]], #4\n" - "fmla v18.4s, bb2.4s, a0.4s\n" - "add %[b_ptr], %[b_ptr], #64\n" - "fmla v19.4s, bb3.4s, a0.4s\n" - "ld1r {a3.4s}, [%[a_ptr3]], #4\n" - - "fmla v20.4s, bb0.4s, a1.4s\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, bb1.4s, a1.4s\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, bb2.4s, a1.4s\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, bb3.4s, a1.4s\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, bb0.4s, a2.4s\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, bb1.4s, a2.4s\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.4s\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, bb3.4s, a2.4s\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, bb0.4s, a3.4s\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, bb1.4s, a3.4s\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, bb2.4s, a3.4s\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, bb3.4s, a3.4s\n" - "str q27, [%[c_ptr2], #48]\n" - - "3:\n" - "str q28, [%[c_ptr3]]\n" - // Increment C pointers for next loop - this looks odd if we - // are using the result buffer, but it's OK as using the - // result buffer implies there will be no next loop. - "add %[c_ptr0], %[c_ptr0], #64\n" - "str q29, [%[c_ptr3], #16]\n" - "add %[c_ptr1], %[c_ptr1], %[a_incr1], LSL #1\n" - "str q30, [%[c_ptr3], #32]\n" - "add %[c_ptr2], %[c_ptr2], %[a_incr2], LSL #1\n" - "str q31, [%[c_ptr3], #48]\n" - "add %[c_ptr3], %[c_ptr3], %[a_incr3], LSL #1\n" - - : [a_ptr0] "+r" (a_ptr0), [a_ptr1] "+r" (a_ptr1), [a_ptr2] "+r" (a_ptr2), [a_ptr3] "+r" (a_ptr3), - [b_ptr] "+r" (b_ptr), [loops] "+r" (loops), [odds] "+r" (odds), - [c_ptr0] "+r" (c_ptr0), [c_ptr1] "+r" (c_ptr1), [c_ptr2] "+r" (c_ptr2), [c_ptr3] "+r" (c_ptr3) - : [oddk] "r" (oddk), [beta0] "r" (beta0), [betaptr] "r" (&beta), - [a_incr1] "r" (a_incr1), [a_incr2] "r" (a_incr2), [a_incr3] "r" (a_incr3) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", - "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", - "cc", "memory" - ); - - /* Copy results from result buffer if needed. */ - if (use_result_buf) { - for (unsigned int row=0; row<active_rows; row++) { - for (unsigned int col=0; col<active_cols; col++) { - C[((y + row) * ldc) + (x0 + col)] = C_buf[row * 16 + col]; - } - } - } - } - } -} - -} // namespace arm_gemm - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp deleted file mode 100644 index 8325b3f352..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4/generic.cpp +++ /dev/null @@ -1,889 +0,0 @@ -/* - * Copyright (c) 2017-2018 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __aarch64__ - -#include <algorithm> -#include <cstddef> -#include <cstring> - -#include <arm_neon.h> - -namespace arm_gemm { - -void a64_sgemm_native_16x4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) { - const int oddk = ((K % 8) >= 4) ? 1 : 0; - const int beta0 = (beta == 0.0f) ? 1 : 0; - const int oddones = (K % 4); - - float dummy_buffer[16]; - - std::memset(dummy_buffer, 0, sizeof(dummy_buffer)); - - /* For now, very naive with no blocking */ - for (int y=0; y<M; y+=4) { - const int activerows = std::min(M-y, 4); - - const float * const a_ptr0_base = A + (y * lda); - const float * const a_ptr1_base = (activerows > 1) ? (a_ptr0_base + lda) : dummy_buffer; - const float * const a_ptr2_base = (activerows > 2) ? (a_ptr1_base + lda) : dummy_buffer; - const float * const a_ptr3_base = (activerows > 3) ? (a_ptr2_base + lda) : dummy_buffer; - - const unsigned long a_incr1 = (activerows > 1) ? 32 : 0; - const unsigned long a_incr2 = (activerows > 2) ? 32 : 0; - const unsigned long a_incr3 = (activerows > 3) ? 32 : 0; - - float *c_ptr0 = C + (y * ldc); - float *c_ptr1 = (activerows > 1) ? c_ptr0 + ldc : dummy_buffer; - float *c_ptr2 = (activerows > 2) ? c_ptr1 + ldc : dummy_buffer; - float *c_ptr3 = (activerows > 3) ? c_ptr2 + ldc : dummy_buffer; - - for (int x0=0; x0<N; x0+=16) { - const float *a_ptr0 = a_ptr0_base; - const float *a_ptr1 = a_ptr1_base; - const float *a_ptr2 = a_ptr2_base; - const float *a_ptr3 = a_ptr3_base; - - const float *b_ptr = B + x0; - - int loops = ((K+4)/8) - 1; - int odds = oddones; - - size_t ldbb = ldb * sizeof(float); - - __asm __volatile ( - "a0 .req v0\n" - "a1 .req v1\n" - "a2 .req v2\n" - "a3 .req v3\n" - "a0a .req v4\n" - "a1a .req v5\n" - "a2a .req v6\n" - "a3a .req v7\n" - "bb0 .req v8\n" - "bb1 .req v9\n" - "bb2 .req v10\n" - "bb3 .req v11\n" - "b0a .req v12\n" - "b1a .req v13\n" - "b2a .req v14\n" - "b3a .req v15\n" - - "a0q .req q0\n" - "a1q .req q1\n" - "a2q .req q2\n" - "a3q .req q3\n" - "a0aq .req q4\n" - "a1aq .req q5\n" - "a2aq .req q6\n" - "a3aq .req q7\n" - "b0q .req q8\n" - "b1q .req q9\n" - "b2q .req q10\n" - "b3q .req q11\n" - "b0aq .req q12\n" - "b1aq .req q13\n" - "b2aq .req q14\n" - "b3aq .req q15\n" - - "movi v16.4s, #0x0\n" - "ldr a0q, [%[a_ptr0]]\n" - "movi v17.4s, #0x0\n" - "ldr b0q, [%[b_ptr]]\n" - "movi v18.4s, #0x0\n" - "ldr b1q, [%[b_ptr], #16]\n" - "movi v19.4s, #0x0\n" - "ldr b2q, [%[b_ptr], #32]\n" - "movi v20.4s, #0x0\n" - "ldr b3q, [%[b_ptr], #48]\n" - "movi v21.4s, #0x0\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "ldr a1q, [%[a_ptr1]]\n" - "movi v22.4s, #0x0\n" - "ldr a2q, [%[a_ptr2]]\n" - "movi v23.4s, #0x0\n" - "ldr a3q, [%[a_ptr3]]\n" - "movi v24.4s, #0x0\n" - "ldr b0aq, [%[b_ptr]]\n" - "movi v25.4s, #0x0\n" - "ldr b1aq, [%[b_ptr], #16]\n" - "movi v26.4s, #0x0\n" - "ldr b2aq, [%[b_ptr], #32]\n" - "cbz %w[beta0], 5f\n" - "movi v27.4s, #0x0\n" - "movi v28.4s, #0x0\n" - "movi v29.4s, #0x0\n" - "movi v30.4s, #0x0\n" - "movi v31.4s, #0x0\n" - - // Skip if no complete loops. - "cbz %w[loops], 4f\n" - "b 1f\n" - - // If beta is non-zero, need to load and multiply by beta - "5:\n" - "ld1r {v4.4s}, [%[betaptr]]\n" - "ldr q16, [%[c_ptr0]]\n" - "ldr q17, [%[c_ptr0], #16]\n" - "ldr q18, [%[c_ptr0], #32]\n" - "ldr q19, [%[c_ptr0], #48]\n" - - "ldr q20, [%[c_ptr1]]\n" - "fmul v16.4s, v16.4s, v4.4s\n" - "ldr q21, [%[c_ptr1], #16]\n" - "fmul v17.4s, v17.4s, v4.4s\n" - "ldr q22, [%[c_ptr1], #32]\n" - "fmul v18.4s, v18.4s, v4.4s\n" - "ldr q23, [%[c_ptr1], #48]\n" - "fmul v19.4s, v19.4s, v4.4s\n" - - "ldr q24, [%[c_ptr2]]\n" - "fmul v20.4s, v20.4s, v4.4s\n" - "ldr q25, [%[c_ptr2], #16]\n" - "fmul v21.4s, v21.4s, v4.4s\n" - "ldr q26, [%[c_ptr2], #32]\n" - "fmul v22.4s, v22.4s, v4.4s\n" - "ldr q27, [%[c_ptr2], #48]\n" - "fmul v23.4s, v23.4s, v4.4s\n" - - "ldr q28, [%[c_ptr3]]\n" - "fmul v24.4s, v24.4s, v4.4s\n" - "ldr q29, [%[c_ptr3], #16]\n" - "fmul v25.4s, v25.4s, v4.4s\n" - "ldr q30, [%[c_ptr3], #32]\n" - "fmul v26.4s, v26.4s, v4.4s\n" - "ldr q31, [%[c_ptr3], #48]\n" - "fmul v27.4s, v27.4s, v4.4s\n" - - "fmul v28.4s, v28.4s, v4.4s\n" - "fmul v29.4s, v29.4s, v4.4s\n" - "fmul v30.4s, v30.4s, v4.4s\n" - "fmul v31.4s, v31.4s, v4.4s\n" - - "cbz %w[loops], 4f\n" - - "1:\n" - // Unroll 0 - "fmla v16.4s, bb0.4s, a0.s[0]\n" - "fmla v20.4s, bb0.4s, a1.s[0]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - "fmla v24.4s, bb0.4s, a2.s[0]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v28.4s, bb0.4s, a3.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[0]\n" - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "ldr a0aq, [%[a_ptr0], #16]\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "ldr a1aq, [%[a_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "ldr a2aq, [%[a_ptr2], #16]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "ldr a3aq, [%[a_ptr3], #16]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "subs %w[loops], %w[loops], #1\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "add %[a_ptr0], %[a_ptr0], #32\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "ldr a0q, [%[a_ptr0]]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 4 - "fmla v16.4s, bb0.4s, a0a.s[0]\n" - "fmla v20.4s, bb0.4s, a1a.s[0]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2a.s[0]\n" - "fmla v28.4s, bb0.4s, a3a.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0a.s[0]\n" - "fmla v21.4s, bb1.4s, a1a.s[0]\n" - "ldr a1q, [%[a_ptr1]]\n" - "fmla v25.4s, bb1.4s, a2a.s[0]\n" - "fmla v29.4s, bb1.4s, a3a.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0a.s[0]\n" - "fmla v22.4s, bb2.4s, a1a.s[0]\n" - "ldr a2q, [%[a_ptr2]]\n" - "fmla v26.4s, bb2.4s, a2a.s[0]\n" - "fmla v30.4s, bb2.4s, a3a.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[0]\n" - "fmla v23.4s, bb3.4s, a1a.s[0]\n" - "ldr a3q, [%[a_ptr3]]\n" - "fmla v27.4s, bb3.4s, a2a.s[0]\n" - "fmla v31.4s, bb3.4s, a3a.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 5 - "fmla v16.4s, b0a.4s, a0a.s[1]\n" - "fmla v20.4s, b0a.4s, a1a.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2a.s[1]\n" - "fmla v28.4s, b0a.4s, a3a.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[1]\n" - "fmla v21.4s, b1a.4s, a1a.s[1]\n" - "fmla v25.4s, b1a.4s, a2a.s[1]\n" - "fmla v29.4s, b1a.4s, a3a.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[1]\n" - "fmla v22.4s, b2a.4s, a1a.s[1]\n" - "fmla v26.4s, b2a.4s, a2a.s[1]\n" - "fmla v30.4s, b2a.4s, a3a.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[1]\n" - "fmla v23.4s, b3a.4s, a1a.s[1]\n" - "fmla v27.4s, b3a.4s, a2a.s[1]\n" - "fmla v31.4s, b3a.4s, a3a.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 6 - "fmla v16.4s, bb0.4s, a0a.s[2]\n" - "fmla v20.4s, bb0.4s, a1a.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2a.s[2]\n" - "fmla v28.4s, bb0.4s, a3a.s[2]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0a.s[2]\n" - "fmla v21.4s, bb1.4s, a1a.s[2]\n" - "fmla v25.4s, bb1.4s, a2a.s[2]\n" - "fmla v29.4s, bb1.4s, a3a.s[2]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0a.s[2]\n" - "fmla v22.4s, bb2.4s, a1a.s[2]\n" - "fmla v26.4s, bb2.4s, a2a.s[2]\n" - "fmla v30.4s, bb2.4s, a3a.s[2]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[2]\n" - "fmla v23.4s, bb3.4s, a1a.s[2]\n" - "fmla v27.4s, bb3.4s, a2a.s[2]\n" - "fmla v31.4s, bb3.4s, a3a.s[2]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 7 - "fmla v16.4s, b0a.4s, a0a.s[3]\n" - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[3]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[3]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[3]\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - "bne 1b\n" - - // Skip to here - "4:\n" - - // Detached final iteration - // Unroll 0 - "fmla v16.4s, bb0.4s, a0.s[0]\n" - "fmla v20.4s, bb0.4s, a1.s[0]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - "fmla v24.4s, bb0.4s, a2.s[0]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v28.4s, bb0.4s, a3.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[0]\n" - "cbnz %w[oddk], 2f\n" // Deal with odd K before we load a0a - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "ldr a0aq, [%[a_ptr0], #16]\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "ldr a1aq, [%[a_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "ldr a2aq, [%[a_ptr2], #16]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "ldr a3aq, [%[a_ptr3], #16]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "add %[a_ptr0], %[a_ptr0], #32\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "add %[a_ptr1], %[a_ptr1], %[a_incr1]\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "add %[a_ptr2], %[a_ptr2], %[a_incr2]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "add %[a_ptr3], %[a_ptr3], %[a_incr3]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 4 - "fmla v16.4s, bb0.4s, a0a.s[0]\n" - "fmla v20.4s, bb0.4s, a1a.s[0]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2a.s[0]\n" - "fmla v28.4s, bb0.4s, a3a.s[0]\n" - "ldr b0q, [%[b_ptr]]\n" - - "fmla v17.4s, bb1.4s, a0a.s[0]\n" - "fmla v21.4s, bb1.4s, a1a.s[0]\n" - "fmla v25.4s, bb1.4s, a2a.s[0]\n" - "fmla v29.4s, bb1.4s, a3a.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0a.s[0]\n" - "fmla v22.4s, bb2.4s, a1a.s[0]\n" - "fmla v26.4s, bb2.4s, a2a.s[0]\n" - "fmla v30.4s, bb2.4s, a3a.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0a.s[0]\n" - "fmla v23.4s, bb3.4s, a1a.s[0]\n" - "fmla v27.4s, bb3.4s, a2a.s[0]\n" - "fmla v31.4s, bb3.4s, a3a.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 5 - "fmla v16.4s, b0a.4s, a0a.s[1]\n" - "fmla v20.4s, b0a.4s, a1a.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, b0a.4s, a2a.s[1]\n" - "fmla v28.4s, b0a.4s, a3a.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0a.s[1]\n" - "fmla v21.4s, b1a.4s, a1a.s[1]\n" - "fmla v25.4s, b1a.4s, a2a.s[1]\n" - "fmla v29.4s, b1a.4s, a3a.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0a.s[1]\n" - "fmla v22.4s, b2a.4s, a1a.s[1]\n" - "fmla v26.4s, b2a.4s, a2a.s[1]\n" - "fmla v30.4s, b2a.4s, a3a.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0a.s[1]\n" - "fmla v23.4s, b3a.4s, a1a.s[1]\n" - "fmla v27.4s, b3a.4s, a2a.s[1]\n" - "fmla v31.4s, b3a.4s, a3a.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 6 - "fmla v16.4s, bb0.4s, a0a.s[2]\n" - "fmla v20.4s, bb0.4s, a1a.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v24.4s, bb0.4s, a2a.s[2]\n" - "fmla v28.4s, bb0.4s, a3a.s[2]\n" - - "fmla v17.4s, bb1.4s, a0a.s[2]\n" - "fmla v21.4s, bb1.4s, a1a.s[2]\n" - "fmla v25.4s, bb1.4s, a2a.s[2]\n" - "fmla v29.4s, bb1.4s, a3a.s[2]\n" - - "fmla v18.4s, bb2.4s, a0a.s[2]\n" - "fmla v22.4s, bb2.4s, a1a.s[2]\n" - "fmla v26.4s, bb2.4s, a2a.s[2]\n" - "fmla v30.4s, bb2.4s, a3a.s[2]\n" - - "fmla v19.4s, bb3.4s, a0a.s[2]\n" - "fmla v23.4s, bb3.4s, a1a.s[2]\n" - "fmla v27.4s, bb3.4s, a2a.s[2]\n" - "fmla v31.4s, bb3.4s, a3a.s[2]\n" - - // Unroll 7 - "fmla v16.4s, b0a.4s, a0a.s[3]\n" - "fmla v17.4s, b1a.4s, a0a.s[3]\n" - "fmla v18.4s, b2a.4s, a0a.s[3]\n" - "fmla v19.4s, b3a.4s, a0a.s[3]\n" - "cbnz %w[odds], 6f\n" - - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - "str q27, [%[c_ptr2], #48]\n" - "b 3f\n" - - // Odd K case: Just do 4 more. - "2:\n" - "fmla v21.4s, bb1.4s, a1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #16\n" - "fmla v25.4s, bb1.4s, a2.s[0]\n" - "add %[a_ptr1], %[a_ptr1], #16\n" - "fmla v29.4s, bb1.4s, a3.s[0]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v18.4s, bb2.4s, a0.s[0]\n" - "add %[a_ptr2], %[a_ptr2], #16\n" - "fmla v22.4s, bb2.4s, a1.s[0]\n" - "add %[a_ptr3], %[a_ptr3], #16\n" - "fmla v26.4s, bb2.4s, a2.s[0]\n" - "fmla v30.4s, bb2.4s, a3.s[0]\n" - "ldr b2q, [%[b_ptr], #32]\n" - - "fmla v19.4s, bb3.4s, a0.s[0]\n" - "fmla v23.4s, bb3.4s, a1.s[0]\n" - "fmla v27.4s, bb3.4s, a2.s[0]\n" - "fmla v31.4s, bb3.4s, a3.s[0]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - // Unroll 1 - "fmla v16.4s, b0a.4s, a0.s[1]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v20.4s, b0a.4s, a1.s[1]\n" - "fmla v24.4s, b0a.4s, a2.s[1]\n" - "fmla v28.4s, b0a.4s, a3.s[1]\n" - "ldr b0aq, [%[b_ptr]]\n" - - "fmla v17.4s, b1a.4s, a0.s[1]\n" - "fmla v21.4s, b1a.4s, a1.s[1]\n" - "fmla v25.4s, b1a.4s, a2.s[1]\n" - "fmla v29.4s, b1a.4s, a3.s[1]\n" - "ldr b1aq, [%[b_ptr], #16]\n" - - "fmla v18.4s, b2a.4s, a0.s[1]\n" - "fmla v22.4s, b2a.4s, a1.s[1]\n" - "fmla v26.4s, b2a.4s, a2.s[1]\n" - "fmla v30.4s, b2a.4s, a3.s[1]\n" - "ldr b2aq, [%[b_ptr], #32]\n" - - "fmla v19.4s, b3a.4s, a0.s[1]\n" - "fmla v23.4s, b3a.4s, a1.s[1]\n" - "fmla v27.4s, b3a.4s, a2.s[1]\n" - "fmla v31.4s, b3a.4s, a3.s[1]\n" - "ldr b3aq, [%[b_ptr], #48]\n" - - // Unroll 2 - "fmla v16.4s, bb0.4s, a0.s[2]\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v20.4s, bb0.4s, a1.s[2]\n" - "fmla v24.4s, bb0.4s, a2.s[2]\n" - "fmla v28.4s, bb0.4s, a3.s[2]\n" - - "fmla v17.4s, bb1.4s, a0.s[2]\n" - "fmla v21.4s, bb1.4s, a1.s[2]\n" - "fmla v25.4s, bb1.4s, a2.s[2]\n" - "fmla v29.4s, bb1.4s, a3.s[2]\n" - - "fmla v18.4s, bb2.4s, a0.s[2]\n" - "fmla v22.4s, bb2.4s, a1.s[2]\n" - "fmla v26.4s, bb2.4s, a2.s[2]\n" - "fmla v30.4s, bb2.4s, a3.s[2]\n" - - "fmla v19.4s, bb3.4s, a0.s[2]\n" - "fmla v23.4s, bb3.4s, a1.s[2]\n" - "fmla v27.4s, bb3.4s, a2.s[2]\n" - "fmla v31.4s, bb3.4s, a3.s[2]\n" - - // Unroll 3 - "fmla v16.4s, b0a.4s, a0.s[3]\n" - "fmla v17.4s, b1a.4s, a0.s[3]\n" - "fmla v18.4s, b2a.4s, a0.s[3]\n" - "fmla v19.4s, b3a.4s, a0.s[3]\n" - "cbnz %w[odds], 7f\n" - - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - "str q27, [%[c_ptr2], #48]\n" - "b 3f\n" - - // "Odd ones" - lead in from even - "6:\n" - "fmla v20.4s, b0a.4s, a1a.s[3]\n" - "fmla v21.4s, b1a.4s, a1a.s[3]\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v22.4s, b2a.4s, a1a.s[3]\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v23.4s, b3a.4s, a1a.s[3]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v24.4s, b0a.4s, a2a.s[3]\n" - "fmla v25.4s, b1a.4s, a2a.s[3]\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v26.4s, b2a.4s, a2a.s[3]\n" - "fmla v27.4s, b3a.4s, a2a.s[3]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - "fmla v28.4s, b0a.4s, a3a.s[3]\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v29.4s, b1a.4s, a3a.s[3]\n" - "fmla v30.4s, b2a.4s, a3a.s[3]\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - "fmla v31.4s, b3a.4s, a3a.s[3]\n" - - "fmla v16.4s, bb0.4s, a0.4s\n" - "beq 9f\n" - "b 8f\n" - - // "Odd ones" - lead in from odd - "7:\n" - "fmla v20.4s, b0a.4s, a1.s[3]\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v21.4s, b1a.4s, a1.s[3]\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v22.4s, b2a.4s, a1.s[3]\n" - "fmla v23.4s, b3a.4s, a1.s[3]\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v24.4s, b0a.4s, a2.s[3]\n" - "fmla v25.4s, b1a.4s, a2.s[3]\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v26.4s, b2a.4s, a2.s[3]\n" - "fmla v27.4s, b3a.4s, a2.s[3]\n" - "ldr b3q, [%[b_ptr], #48]\n" - - "fmla v28.4s, b0a.4s, a3.s[3]\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v29.4s, b1a.4s, a3.s[3]\n" - "fmla v30.4s, b2a.4s, a3.s[3]\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - "fmla v31.4s, b3a.4s, a3.s[3]\n" - - "fmla v16.4s, bb0.4s, a0.4s\n" - "beq 9f\n" - - // "Odd ones" - loop - "8:\n" - "fmla v17.4s, bb1.4s, a0.4s\n" - "ld1r {a2.4s}, [%[a_ptr2]], #4\n" - "fmla v18.4s, bb2.4s, a0.4s\n" - "add %[b_ptr], %[b_ptr], %[ldb]\n" - "fmla v19.4s, bb3.4s, a0.4s\n" - "ld1r {a3.4s}, [%[a_ptr3]], #4\n" - - "fmla v20.4s, bb0.4s, a1.4s\n" - "subs %w[odds], %w[odds], #1\n" - "fmla v21.4s, bb1.4s, a1.4s\n" - "ld1r {a0.4s}, [%[a_ptr0]], #4\n" - "fmla v22.4s, bb2.4s, a1.4s\n" - "fmla v23.4s, bb3.4s, a1.4s\n" - "ld1r {a1.4s}, [%[a_ptr1]], #4\n" - - "fmla v24.4s, bb0.4s, a2.4s\n" - "fmla v28.4s, bb0.4s, a3.4s\n" - "ldr b0q, [%[b_ptr]]\n" - "fmla v25.4s, bb1.4s, a2.4s\n" - "fmla v29.4s, bb1.4s, a3.4s\n" - "ldr b1q, [%[b_ptr], #16]\n" - - "fmla v26.4s, bb2.4s, a2.4s\n" - "fmla v30.4s, bb2.4s, a3.4s\n" - "ldr b2q, [%[b_ptr], #32]\n" - "fmla v27.4s, bb3.4s, a2.4s\n" - "fmla v31.4s, bb3.4s, a3.4s\n" - "ldr b3q, [%[b_ptr], #48]\n" - "fmla v16.4s, bb0.4s, a0.4s\n" - "bne 8b\n" - - // "Odd ones" - detached final iteration - "9:\n" - "fmla v17.4s, bb1.4s, a0.4s\n" - "ld1r {a2.4s}, [%[a_ptr2]], #4\n" - "fmla v18.4s, bb2.4s, a0.4s\n" - "fmla v19.4s, bb3.4s, a0.4s\n" - "ld1r {a3.4s}, [%[a_ptr3]], #4\n" - - "fmla v20.4s, bb0.4s, a1.4s\n" - "str q16, [%[c_ptr0]]\n" - "fmla v21.4s, bb1.4s, a1.4s\n" - "str q17, [%[c_ptr0], #16]\n" - "fmla v22.4s, bb2.4s, a1.4s\n" - "str q18, [%[c_ptr0], #32]\n" - "fmla v23.4s, bb3.4s, a1.4s\n" - "str q19, [%[c_ptr0], #48]\n" - - "fmla v24.4s, bb0.4s, a2.4s\n" - "str q20, [%[c_ptr1]]\n" - "fmla v25.4s, bb1.4s, a2.4s\n" - "str q21, [%[c_ptr1], #16]\n" - "fmla v26.4s, bb2.4s, a2.4s\n" - "str q22, [%[c_ptr1], #32]\n" - "fmla v27.4s, bb3.4s, a2.4s\n" - "str q23, [%[c_ptr1], #48]\n" - - "fmla v28.4s, bb0.4s, a3.4s\n" - "str q24, [%[c_ptr2]]\n" - "fmla v29.4s, bb1.4s, a3.4s\n" - "str q25, [%[c_ptr2], #16]\n" - "fmla v30.4s, bb2.4s, a3.4s\n" - "str q26, [%[c_ptr2], #32]\n" - "fmla v31.4s, bb3.4s, a3.4s\n" - "str q27, [%[c_ptr2], #48]\n" - - "3:\n" - "str q28, [%[c_ptr3]]\n" - "add %[c_ptr0], %[c_ptr0], #64\n" - "str q29, [%[c_ptr3], #16]\n" - "add %[c_ptr1], %[c_ptr1], %[a_incr1], LSL #1\n" - "str q30, [%[c_ptr3], #32]\n" - "add %[c_ptr2], %[c_ptr2], %[a_incr2], LSL #1\n" - "str q31, [%[c_ptr3], #48]\n" - "add %[c_ptr3], %[c_ptr3], %[a_incr3], LSL #1\n" - - : [a_ptr0] "+r" (a_ptr0), [a_ptr1] "+r" (a_ptr1), [a_ptr2] "+r" (a_ptr2), [a_ptr3] "+r" (a_ptr3), - [b_ptr] "+r" (b_ptr), [loops] "+r" (loops), [odds] "+r" (odds), - [c_ptr0] "+r" (c_ptr0), [c_ptr1] "+r" (c_ptr1), [c_ptr2] "+r" (c_ptr2), [c_ptr3] "+r" (c_ptr3) - : [ldb] "r" (ldbb), [oddk] "r" (oddk), [beta0] "r" (beta0), [betaptr] "r" (&beta), - [a_incr1] "r" (a_incr1), [a_incr2] "r" (a_incr2), [a_incr3] "r" (a_incr3) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", - "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", - "cc", "memory" - ); - } - } -} - -} // namespace arm_gemm - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp index 022efdfc26..352a147282 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -23,51 +23,61 @@ */ #pragma once -#ifdef __ARM_FEATURE_SVE - - +#ifdef __aarch64__ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_hybrid_fp32_mla_1VLx4(const float *, int, const float *, float *, int, float, int, int, int); +void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); -class smallK_hybrid_fp32_mla_1VLx4 +class smallK_hybrid_fp32_mla_4x6 { public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { - return 4; + return 6; } static unsigned int out_width() { - return get_vector_length<float>() * 1; + return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } - StdTransformsSVE<operand_type, result_type, 4, 1, 1> transforms = {}; - - // Default to the generic kernel - kern_type kernel=sve_smallK_hybrid_fp32_mla_1VLx4; + static constexpr bool supports_append() + { + return false; + } - smallK_hybrid_fp32_mla_1VLx4(const CPUInfo *ci) + static constexpr bool supports_bias() { + return true; + } + static constexpr bool supports_activation() + { + return true; } + + StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {}; + + // Default to the generic kernel + kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6; + + smallK_hybrid_fp32_mla_4x6(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp new file mode 100644 index 0000000000..d8e8e52417 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp @@ -0,0 +1,4612 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __aarch64__ + +#include <algorithm> + +#include "arm_gemm.hpp" + + +#include "../../asmlib.hpp" +#include "../../utils.hpp" + +namespace arm_gemm { + +void a64_smallK_hybrid_fp32_mla_4x6(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; + const long ldab = lda * sizeof(float); + const long ldcb = ldc * sizeof(float); + float nullbias[4]; + if (!bias) { + memset(nullbias, 0, (4 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + for (int y0=0; y0<M; y0+=6) { + long loops = loops_count; + long oob_rows = std::max(6 - (M-y0), 0); + const float *b_ptr0 = B; + const float *biasptr = bias ? bias : nullbias; + const uint64_t biasinc = bias ? 4*sizeof(float) : 0; + const float *a_ptr0 = A + (y0 * lda); + + float *c_ptr0 = C + (y0 * ldc); + + switch(K) { + case 9: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr s2, [%[a_ptr0]]\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr s5, [a_ptr1]\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr s8, [a_ptr2]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr s11, [a_ptr3]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr s14, [a_ptr4]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr s17, [a_ptr5]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 10: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr d2, [%[a_ptr0]]\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1]\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr d11, [a_ptr3]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr d14, [a_ptr4]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr d17, [a_ptr5]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 11: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr d2, [%[a_ptr0]], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr d17, [a_ptr5], #0x8\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ld1 {v2.s}[2], [%[a_ptr0]]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 12: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q11, [a_ptr3]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q14, [a_ptr4]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q17, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "ldr q18, [%[b_ptr0]]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "fmla v26.4s, v21.4s, v2.s[3]\n" + "fmla v27.4s, v21.4s, v5.s[3]\n" + "fmla v28.4s, v21.4s, v8.s[3]\n" + "fmla v29.4s, v21.4s, v11.s[3]\n" + "fmla v30.4s, v21.4s, v14.s[3]\n" + "fmla v31.4s, v21.4s, v17.s[3]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "fmla v26.4s, v21.4s, v2.s[3]\n" + "fmla v27.4s, v21.4s, v5.s[3]\n" + "fmla v28.4s, v21.4s, v8.s[3]\n" + "fmla v29.4s, v21.4s, v11.s[3]\n" + "fmla v30.4s, v21.4s, v14.s[3]\n" + "fmla v31.4s, v21.4s, v17.s[3]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "ldr q18, [%[b_ptr0]]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "fmla v26.4s, v21.4s, v2.s[3]\n" + "fmla v27.4s, v21.4s, v5.s[3]\n" + "fmla v28.4s, v21.4s, v8.s[3]\n" + "fmla v29.4s, v21.4s, v11.s[3]\n" + "fmla v30.4s, v21.4s, v14.s[3]\n" + "fmla v31.4s, v21.4s, v17.s[3]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v18.4s, v0.s[0]\n" + "fmla v27.4s, v18.4s, v3.s[0]\n" + "fmla v28.4s, v18.4s, v6.s[0]\n" + "fmla v29.4s, v18.4s, v9.s[0]\n" + "fmla v30.4s, v18.4s, v12.s[0]\n" + "fmla v31.4s, v18.4s, v15.s[0]\n" + "ldr q18, [%[b_ptr0]]\n" + "fmla v26.4s, v19.4s, v0.s[1]\n" + "fmla v27.4s, v19.4s, v3.s[1]\n" + "fmla v28.4s, v19.4s, v6.s[1]\n" + "fmla v29.4s, v19.4s, v9.s[1]\n" + "fmla v30.4s, v19.4s, v12.s[1]\n" + "fmla v31.4s, v19.4s, v15.s[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v20.4s, v0.s[2]\n" + "fmla v27.4s, v20.4s, v3.s[2]\n" + "fmla v28.4s, v20.4s, v6.s[2]\n" + "fmla v29.4s, v20.4s, v9.s[2]\n" + "fmla v30.4s, v20.4s, v12.s[2]\n" + "fmla v31.4s, v20.4s, v15.s[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "fmla v26.4s, v21.4s, v0.s[3]\n" + "fmla v27.4s, v21.4s, v3.s[3]\n" + "fmla v28.4s, v21.4s, v6.s[3]\n" + "fmla v29.4s, v21.4s, v9.s[3]\n" + "fmla v30.4s, v21.4s, v12.s[3]\n" + "fmla v31.4s, v21.4s, v15.s[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "fmla v26.4s, v22.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmla v27.4s, v22.4s, v4.s[0]\n" + "fmla v28.4s, v22.4s, v7.s[0]\n" + "fmla v29.4s, v22.4s, v10.s[0]\n" + "fmla v30.4s, v22.4s, v13.s[0]\n" + "fmla v31.4s, v22.4s, v16.s[0]\n" + "fmla v26.4s, v23.4s, v1.s[1]\n" + "fmla v27.4s, v23.4s, v4.s[1]\n" + "fmla v28.4s, v23.4s, v7.s[1]\n" + "fmla v29.4s, v23.4s, v10.s[1]\n" + "fmla v30.4s, v23.4s, v13.s[1]\n" + "fmla v31.4s, v23.4s, v16.s[1]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v7.s[2]\n" + "fmla v29.4s, v24.4s, v10.s[2]\n" + "fmla v30.4s, v24.4s, v13.s[2]\n" + "fmla v31.4s, v24.4s, v16.s[2]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v7.s[3]\n" + "fmla v29.4s, v25.4s, v10.s[3]\n" + "fmla v30.4s, v25.4s, v13.s[3]\n" + "fmla v31.4s, v25.4s, v16.s[3]\n" + "fmla v26.4s, v18.4s, v2.s[0]\n" + "fmla v27.4s, v18.4s, v5.s[0]\n" + "fmla v28.4s, v18.4s, v8.s[0]\n" + "fmla v29.4s, v18.4s, v11.s[0]\n" + "fmla v30.4s, v18.4s, v14.s[0]\n" + "fmla v31.4s, v18.4s, v17.s[0]\n" + "fmla v26.4s, v19.4s, v2.s[1]\n" + "fmla v27.4s, v19.4s, v5.s[1]\n" + "fmla v28.4s, v19.4s, v8.s[1]\n" + "fmla v29.4s, v19.4s, v11.s[1]\n" + "fmla v30.4s, v19.4s, v14.s[1]\n" + "fmla v31.4s, v19.4s, v17.s[1]\n" + "fmla v26.4s, v20.4s, v2.s[2]\n" + "fmla v27.4s, v20.4s, v5.s[2]\n" + "fmla v28.4s, v20.4s, v8.s[2]\n" + "fmla v29.4s, v20.4s, v11.s[2]\n" + "fmla v30.4s, v20.4s, v14.s[2]\n" + "fmla v31.4s, v20.4s, v17.s[2]\n" + "fmla v26.4s, v21.4s, v2.s[3]\n" + "fmla v27.4s, v21.4s, v5.s[3]\n" + "fmla v28.4s, v21.4s, v8.s[3]\n" + "fmla v29.4s, v21.4s, v11.s[3]\n" + "fmla v30.4s, v21.4s, v14.s[3]\n" + "fmla v31.4s, v21.4s, v17.s[3]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 13: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q8, [a_ptr2], #0x10\n" + "ldr q12, [a_ptr3], #0x10\n" + "ldr q16, [a_ptr4], #0x10\n" + "ldr q20, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q5, [a_ptr1], #0x10\n" + "ldr q9, [a_ptr2], #0x10\n" + "ldr q13, [a_ptr3], #0x10\n" + "ldr q17, [a_ptr4], #0x10\n" + "ldr q21, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]], #0x10\n" + "ldr q6, [a_ptr1], #0x10\n" + "ldr q10, [a_ptr2], #0x10\n" + "ldr q14, [a_ptr3], #0x10\n" + "ldr s3, [%[a_ptr0]]\n" + "ldr q18, [a_ptr4], #0x10\n" + "ldr s7, [a_ptr1]\n" + "ldr q22, [a_ptr5], #0x10\n" + "ldr s11, [a_ptr2]\n" + "ldr q24, [%[b_ptr0]]\n" + "ldr s15, [a_ptr3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr s19, [a_ptr4]\n" + "ldr s23, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 14: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q8, [a_ptr2], #0x10\n" + "ldr q12, [a_ptr3], #0x10\n" + "ldr q16, [a_ptr4], #0x10\n" + "ldr q20, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q5, [a_ptr1], #0x10\n" + "ldr q9, [a_ptr2], #0x10\n" + "ldr q13, [a_ptr3], #0x10\n" + "ldr q17, [a_ptr4], #0x10\n" + "ldr q21, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]], #0x10\n" + "ldr q6, [a_ptr1], #0x10\n" + "ldr q10, [a_ptr2], #0x10\n" + "ldr q14, [a_ptr3], #0x10\n" + "ldr d3, [%[a_ptr0]]\n" + "ldr q18, [a_ptr4], #0x10\n" + "ldr d7, [a_ptr1]\n" + "ldr q22, [a_ptr5], #0x10\n" + "ldr d11, [a_ptr2]\n" + "ldr q24, [%[b_ptr0]]\n" + "ldr d15, [a_ptr3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr d19, [a_ptr4]\n" + "ldr d23, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 15: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q8, [a_ptr2], #0x10\n" + "ldr q12, [a_ptr3], #0x10\n" + "ldr q16, [a_ptr4], #0x10\n" + "ldr q20, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q5, [a_ptr1], #0x10\n" + "ldr q9, [a_ptr2], #0x10\n" + "ldr q13, [a_ptr3], #0x10\n" + "ldr q17, [a_ptr4], #0x10\n" + "ldr q21, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]], #0x10\n" + "ldr q6, [a_ptr1], #0x10\n" + "ldr q10, [a_ptr2], #0x10\n" + "ldr q14, [a_ptr3], #0x10\n" + "ldr d3, [%[a_ptr0]], #0x8\n" + "ldr q18, [a_ptr4], #0x10\n" + "ldr d7, [a_ptr1], #0x8\n" + "ldr q22, [a_ptr5], #0x10\n" + "ldr d11, [a_ptr2], #0x8\n" + "ldr q24, [%[b_ptr0]]\n" + "ldr d15, [a_ptr3], #0x8\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr d19, [a_ptr4], #0x8\n" + "ldr d23, [a_ptr5], #0x8\n" + "ld1 {v3.s}[2], [%[a_ptr0]]\n" + "ld1 {v7.s}[2], [a_ptr1]\n" + "ld1 {v11.s}[2], [a_ptr2]\n" + "ld1 {v15.s}[2], [a_ptr3]\n" + "ld1 {v19.s}[2], [a_ptr4]\n" + "ld1 {v23.s}[2], [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + default: + case 16: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "c_ptr1 .req X5\n" + "c_ptr2 .req X6\n" + "c_ptr3 .req X7\n" + "c_ptr4 .req X8\n" + "c_ptr5 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q8, [a_ptr2], #0x10\n" + "ldr q12, [a_ptr3], #0x10\n" + "ldr q16, [a_ptr4], #0x10\n" + "ldr q20, [a_ptr5], #0x10\n" + "ldr q1, [%[a_ptr0]], #0x10\n" + "ldr q5, [a_ptr1], #0x10\n" + "ldr q9, [a_ptr2], #0x10\n" + "ldr q13, [a_ptr3], #0x10\n" + "ldr q17, [a_ptr4], #0x10\n" + "ldr q21, [a_ptr5], #0x10\n" + "ldr q2, [%[a_ptr0]], #0x10\n" + "ldr q6, [a_ptr1], #0x10\n" + "ldr q10, [a_ptr2], #0x10\n" + "ldr q14, [a_ptr3], #0x10\n" + "ldr q18, [a_ptr4], #0x10\n" + "ldr q22, [a_ptr5], #0x10\n" + "ldr q3, [%[a_ptr0]]\n" + "ldr q7, [a_ptr1]\n" + "ldr q11, [a_ptr2]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q15, [a_ptr3]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q19, [a_ptr4]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q23, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" + "ldr q24, [%[b_ptr0]]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "fmla v26.4s, v25.4s, v3.s[3]\n" + "fmla v27.4s, v25.4s, v7.s[3]\n" + "fmla v28.4s, v25.4s, v11.s[3]\n" + "fmla v29.4s, v25.4s, v15.s[3]\n" + "fmla v30.4s, v25.4s, v19.s[3]\n" + "fmla v31.4s, v25.4s, v23.s[3]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "fmla v26.4s, v25.4s, v3.s[3]\n" + "fmla v27.4s, v25.4s, v7.s[3]\n" + "fmla v28.4s, v25.4s, v11.s[3]\n" + "fmla v29.4s, v25.4s, v15.s[3]\n" + "fmla v30.4s, v25.4s, v19.s[3]\n" + "fmla v31.4s, v25.4s, v23.s[3]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "ldr q26, [%[biasptr]]\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "ldr q24, [%[b_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "mov v27.16b, v26.16b\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "mov v28.16b, v26.16b\n" + "add c_ptr2, c_ptr2, #0x10\n" + "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v29.16b, v26.16b\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "str q30, [c_ptr4]\n" + "mov v30.16b, v26.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "str q31, [c_ptr5]\n" + "mov v31.16b, v26.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "fmla v26.4s, v25.4s, v3.s[3]\n" + "fmla v27.4s, v25.4s, v7.s[3]\n" + "fmla v28.4s, v25.4s, v11.s[3]\n" + "fmla v29.4s, v25.4s, v15.s[3]\n" + "fmla v30.4s, v25.4s, v19.s[3]\n" + "fmla v31.4s, v25.4s, v23.s[3]\n" + "b 5f\n" + "2:\n" + "ldr q26, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v27.16b, v26.16b\n" + "mov v28.16b, v26.16b\n" + "mov v29.16b, v26.16b\n" + "mov v30.16b, v26.16b\n" + "mov v31.16b, v26.16b\n" + "fmla v26.4s, v24.4s, v0.s[0]\n" + "fmla v27.4s, v24.4s, v4.s[0]\n" + "fmla v28.4s, v24.4s, v8.s[0]\n" + "fmla v29.4s, v24.4s, v12.s[0]\n" + "fmla v30.4s, v24.4s, v16.s[0]\n" + "fmla v31.4s, v24.4s, v20.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[1]\n" + "fmla v27.4s, v25.4s, v4.s[1]\n" + "fmla v28.4s, v25.4s, v8.s[1]\n" + "fmla v29.4s, v25.4s, v12.s[1]\n" + "fmla v30.4s, v25.4s, v16.s[1]\n" + "fmla v31.4s, v25.4s, v20.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v0.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v4.s[2]\n" + "fmla v28.4s, v24.4s, v8.s[2]\n" + "fmla v29.4s, v24.4s, v12.s[2]\n" + "fmla v30.4s, v24.4s, v16.s[2]\n" + "fmla v31.4s, v24.4s, v20.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v0.s[3]\n" + "fmla v27.4s, v25.4s, v4.s[3]\n" + "fmla v28.4s, v25.4s, v8.s[3]\n" + "fmla v29.4s, v25.4s, v12.s[3]\n" + "fmla v30.4s, v25.4s, v16.s[3]\n" + "fmla v31.4s, v25.4s, v20.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[0]\n" + "fmla v28.4s, v24.4s, v9.s[0]\n" + "fmla v29.4s, v24.4s, v13.s[0]\n" + "fmla v30.4s, v24.4s, v17.s[0]\n" + "fmla v31.4s, v24.4s, v21.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[1]\n" + "fmla v27.4s, v25.4s, v5.s[1]\n" + "fmla v28.4s, v25.4s, v9.s[1]\n" + "fmla v29.4s, v25.4s, v13.s[1]\n" + "fmla v30.4s, v25.4s, v17.s[1]\n" + "fmla v31.4s, v25.4s, v21.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v1.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v5.s[2]\n" + "fmla v28.4s, v24.4s, v9.s[2]\n" + "fmla v29.4s, v24.4s, v13.s[2]\n" + "fmla v30.4s, v24.4s, v17.s[2]\n" + "fmla v31.4s, v24.4s, v21.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v1.s[3]\n" + "fmla v27.4s, v25.4s, v5.s[3]\n" + "fmla v28.4s, v25.4s, v9.s[3]\n" + "fmla v29.4s, v25.4s, v13.s[3]\n" + "fmla v30.4s, v25.4s, v17.s[3]\n" + "fmla v31.4s, v25.4s, v21.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[0]\n" + "fmla v28.4s, v24.4s, v10.s[0]\n" + "fmla v29.4s, v24.4s, v14.s[0]\n" + "fmla v30.4s, v24.4s, v18.s[0]\n" + "fmla v31.4s, v24.4s, v22.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[1]\n" + "fmla v27.4s, v25.4s, v6.s[1]\n" + "fmla v28.4s, v25.4s, v10.s[1]\n" + "fmla v29.4s, v25.4s, v14.s[1]\n" + "fmla v30.4s, v25.4s, v18.s[1]\n" + "fmla v31.4s, v25.4s, v22.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v2.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v6.s[2]\n" + "fmla v28.4s, v24.4s, v10.s[2]\n" + "fmla v29.4s, v24.4s, v14.s[2]\n" + "fmla v30.4s, v24.4s, v18.s[2]\n" + "fmla v31.4s, v24.4s, v22.s[2]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v2.s[3]\n" + "fmla v27.4s, v25.4s, v6.s[3]\n" + "fmla v28.4s, v25.4s, v10.s[3]\n" + "fmla v29.4s, v25.4s, v14.s[3]\n" + "fmla v30.4s, v25.4s, v18.s[3]\n" + "fmla v31.4s, v25.4s, v22.s[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[0]\n" + "fmla v28.4s, v24.4s, v11.s[0]\n" + "fmla v29.4s, v24.4s, v15.s[0]\n" + "fmla v30.4s, v24.4s, v19.s[0]\n" + "fmla v31.4s, v24.4s, v23.s[0]\n" + "ldr q24, [%[b_ptr0]]\n" + "fmla v26.4s, v25.4s, v3.s[1]\n" + "fmla v27.4s, v25.4s, v7.s[1]\n" + "fmla v28.4s, v25.4s, v11.s[1]\n" + "fmla v29.4s, v25.4s, v15.s[1]\n" + "fmla v30.4s, v25.4s, v19.s[1]\n" + "fmla v31.4s, v25.4s, v23.s[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + "fmla v26.4s, v24.4s, v3.s[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmla v27.4s, v24.4s, v7.s[2]\n" + "fmla v28.4s, v24.4s, v11.s[2]\n" + "fmla v29.4s, v24.4s, v15.s[2]\n" + "fmla v30.4s, v24.4s, v19.s[2]\n" + "fmla v31.4s, v24.4s, v23.s[2]\n" + "fmla v26.4s, v25.4s, v3.s[3]\n" + "fmla v27.4s, v25.4s, v7.s[3]\n" + "fmla v28.4s, v25.4s, v11.s[3]\n" + "fmla v29.4s, v25.4s, v15.s[3]\n" + "fmla v30.4s, v25.4s, v19.s[3]\n" + "fmla v31.4s, v25.4s, v23.s[3]\n" + "5:\n" + "ld1r {v24.4s}, [%[minptr]]\n" + "ld1r {v25.4s}, [%[maxptr]]\n" + "fmax v26.4s, v26.4s, v24.4s\n" + "fmax v27.4s, v27.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v24.4s\n" + "fmax v29.4s, v29.4s, v24.4s\n" + "fmin v26.4s, v26.4s, v25.4s\n" + "fmin v27.4s, v27.4s, v25.4s\n" + "fmin v28.4s, v28.4s, v25.4s\n" + "fmin v29.4s, v29.4s, v25.4s\n" + "str q26, [%[c_ptr0]]\n" + "fmax v30.4s, v30.4s, v24.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v31.4s, v31.4s, v24.4s\n" + "str q27, [c_ptr1]\n" + "fmin v30.4s, v30.4s, v25.4s\n" + "fmin v31.4s, v31.4s, v25.4s\n" + "str q28, [c_ptr2]\n" + "str q29, [c_ptr3]\n" + "str q30, [c_ptr4]\n" + "str q31, [c_ptr5]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + } + } +} + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp index a5511931d8..c5d39cbc87 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_native_16x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Arm Limited. + * Copyright (c) 2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -25,45 +25,59 @@ #ifdef __aarch64__ -namespace arm_gemm { + + +namespace arm_gemm +{ // Actual kernel implementations -void a64_sgemm_native_16x4(const float *, int, const float *, int, float *, int, float, int, int, int); - -// 12x8 SGEMM "strategy" class. -// -// This describes the characteristics of a family of kernels, in terms of -// the required interleave properties and the output block size. -// -// All kernels in the family must share these characteristics. The actual -// kernel to be used can be chosen at runtime, based on the CPU_type -// structure. -class sgemm_native_16x4 { +void a64_smallK_hybrid_fp32_mla_4x8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); + +class smallK_hybrid_fp32_mla_4x8 +{ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, int, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_width() { - return 16; + static constexpr unsigned int out_height() + { + return 8; } - static unsigned int out_height() { + static unsigned int out_width() + { return 4; } - static unsigned int k_unroll() { + static constexpr unsigned int k_unroll() + { return 1; } - // Default to the generic kernel - kern_type kernel=a64_sgemm_native_16x4; + static constexpr bool supports_append() + { + return false; + } - sgemm_native_16x4(const CPUInfo *ci) { - UNUSED(ci); + static constexpr bool supports_bias() + { + return true; } + + static constexpr bool supports_activation() + { + return true; + } + + StdTransformsFixed<operand_type, result_type, 8, 4, 1> transforms = {}; + + // Default to the generic kernel + kern_type kernel=a64_smallK_hybrid_fp32_mla_4x8; + + smallK_hybrid_fp32_mla_4x8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp new file mode 100644 index 0000000000..7ad52059b6 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp @@ -0,0 +1,3340 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __aarch64__ + +#include <algorithm> + +#include "arm_gemm.hpp" + + +#include "../../asmlib.hpp" +#include "../../utils.hpp" + +namespace arm_gemm { + +void a64_smallK_hybrid_fp32_mla_4x8(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; + const long ldab = lda * sizeof(float); + const long ldcb = ldc * sizeof(float); + float nullbias[4]; + if (!bias) { + memset(nullbias, 0, (4 * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } + + for (int y0=0; y0<M; y0+=8) { + long loops = loops_count; + long oob_rows = std::max(8 - (M-y0), 0); + const float *b_ptr0 = B; + const float *biasptr = bias ? bias : nullbias; + const uint64_t biasinc = bias ? 4*sizeof(float) : 0; + const float *a_ptr0 = A + (y0 * lda); + + float *c_ptr0 = C + (y0 * ldc); + + switch(K) { + case 1: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr s0, [%[a_ptr0]]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 2: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr d0, [%[a_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 3: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr d0, [%[a_ptr0]], #0x8\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" + "ld1 {v0.s}[2], [%[a_ptr0]]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 4: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "ldr q1, [a_ptr1]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "ldr q2, [a_ptr2]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "ldr q3, [a_ptr3]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "ldr q4, [a_ptr4]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "ldr q5, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v1.s[3]\n" + "fmla v26.4s, v19.4s, v2.s[3]\n" + "fmla v27.4s, v19.4s, v3.s[3]\n" + "fmla v28.4s, v19.4s, v4.s[3]\n" + "fmla v29.4s, v19.4s, v5.s[3]\n" + "fmla v30.4s, v19.4s, v6.s[3]\n" + "fmla v31.4s, v19.4s, v7.s[3]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v1.s[3]\n" + "fmla v26.4s, v19.4s, v2.s[3]\n" + "fmla v27.4s, v19.4s, v3.s[3]\n" + "fmla v28.4s, v19.4s, v4.s[3]\n" + "fmla v29.4s, v19.4s, v5.s[3]\n" + "fmla v30.4s, v19.4s, v6.s[3]\n" + "fmla v31.4s, v19.4s, v7.s[3]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v1.s[3]\n" + "fmla v26.4s, v19.4s, v2.s[3]\n" + "fmla v27.4s, v19.4s, v3.s[3]\n" + "fmla v28.4s, v19.4s, v4.s[3]\n" + "fmla v29.4s, v19.4s, v5.s[3]\n" + "fmla v30.4s, v19.4s, v6.s[3]\n" + "fmla v31.4s, v19.4s, v7.s[3]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v1.s[0]\n" + "fmla v26.4s, v16.4s, v2.s[0]\n" + "fmla v27.4s, v16.4s, v3.s[0]\n" + "fmla v28.4s, v16.4s, v4.s[0]\n" + "fmla v29.4s, v16.4s, v5.s[0]\n" + "fmla v30.4s, v16.4s, v6.s[0]\n" + "fmla v31.4s, v16.4s, v7.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v1.s[1]\n" + "fmla v26.4s, v17.4s, v2.s[1]\n" + "fmla v27.4s, v17.4s, v3.s[1]\n" + "fmla v28.4s, v17.4s, v4.s[1]\n" + "fmla v29.4s, v17.4s, v5.s[1]\n" + "fmla v30.4s, v17.4s, v6.s[1]\n" + "fmla v31.4s, v17.4s, v7.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v1.s[2]\n" + "fmla v26.4s, v18.4s, v2.s[2]\n" + "fmla v27.4s, v18.4s, v3.s[2]\n" + "fmla v28.4s, v18.4s, v4.s[2]\n" + "fmla v29.4s, v18.4s, v5.s[2]\n" + "fmla v30.4s, v18.4s, v6.s[2]\n" + "fmla v31.4s, v18.4s, v7.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v1.s[3]\n" + "fmla v26.4s, v19.4s, v2.s[3]\n" + "fmla v27.4s, v19.4s, v3.s[3]\n" + "fmla v28.4s, v19.4s, v4.s[3]\n" + "fmla v29.4s, v19.4s, v5.s[3]\n" + "fmla v30.4s, v19.4s, v6.s[3]\n" + "fmla v31.4s, v19.4s, v7.s[3]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 5: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q2, [a_ptr1], #0x10\n" + "ldr q4, [a_ptr2], #0x10\n" + "ldr q6, [a_ptr3], #0x10\n" + "ldr s1, [%[a_ptr0]]\n" + "ldr q8, [a_ptr4], #0x10\n" + "ldr s3, [a_ptr1]\n" + "ldr q10, [a_ptr5], #0x10\n" + "ldr s5, [a_ptr2]\n" + "ldr q12, [a_ptr6], #0x10\n" + "ldr s7, [a_ptr3]\n" + "ldr q14, [a_ptr7], #0x10\n" + "ldr s9, [a_ptr4]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr s11, [a_ptr5]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr s13, [a_ptr6]\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr s15, [a_ptr7]\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x50\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x50\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v25.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x50\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v26.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v27.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v29.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 6: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q2, [a_ptr1], #0x10\n" + "ldr q4, [a_ptr2], #0x10\n" + "ldr q6, [a_ptr3], #0x10\n" + "ldr d1, [%[a_ptr0]]\n" + "ldr q8, [a_ptr4], #0x10\n" + "ldr d3, [a_ptr1]\n" + "ldr q10, [a_ptr5], #0x10\n" + "ldr d5, [a_ptr2]\n" + "ldr q12, [a_ptr6], #0x10\n" + "ldr d7, [a_ptr3]\n" + "ldr q14, [a_ptr7], #0x10\n" + "ldr d9, [a_ptr4]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr d11, [a_ptr5]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr d13, [a_ptr6]\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr d15, [a_ptr7]\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x60\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x60\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v25.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "add %[b_ptr0], %[b_ptr0], #0x60\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "add c_ptr3, c_ptr3, #0x10\n" + "mov v25.16b, v24.16b\n" + "str q28, [c_ptr4]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q29, [c_ptr5]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q30, [c_ptr6]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "mov v30.16b, v24.16b\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + case 7: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q2, [a_ptr1], #0x10\n" + "ldr q4, [a_ptr2], #0x10\n" + "ldr q6, [a_ptr3], #0x10\n" + "ldr d1, [%[a_ptr0]], #0x8\n" + "ldr q8, [a_ptr4], #0x10\n" + "ldr d3, [a_ptr1], #0x8\n" + "ldr q10, [a_ptr5], #0x10\n" + "ldr d5, [a_ptr2], #0x8\n" + "ldr q12, [a_ptr6], #0x10\n" + "ldr d7, [a_ptr3], #0x8\n" + "ldr q14, [a_ptr7], #0x10\n" + "ldr d9, [a_ptr4], #0x8\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr d11, [a_ptr5], #0x8\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr d13, [a_ptr6], #0x8\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr d15, [a_ptr7], #0x8\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "ld1 {v1.s}[2], [%[a_ptr0]]\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "ld1 {v3.s}[2], [a_ptr1]\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "ld1 {v5.s}[2], [a_ptr2]\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "ld1 {v7.s}[2], [a_ptr3]\n" + "ld1 {v9.s}[2], [a_ptr4]\n" + "ld1 {v11.s}[2], [a_ptr5]\n" + "ld1 {v13.s}[2], [a_ptr6]\n" + "ld1 {v15.s}[2], [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "add %[b_ptr0], %[b_ptr0], #0x70\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x70\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "mov v27.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "mov v28.16b, v24.16b\n" + "str q29, [c_ptr5]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x70\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + default: + case 8: + __asm __volatile ( + "a_ptr1 .req X0\n" + "a_ptr2 .req X1\n" + "a_ptr3 .req X2\n" + "a_ptr4 .req X3\n" + "a_ptr5 .req X4\n" + "a_ptr6 .req X5\n" + "a_ptr7 .req X6\n" + "c_ptr1 .req X7\n" + "c_ptr2 .req X8\n" + "c_ptr3 .req X9\n" + "c_ptr4 .req X10\n" + "c_ptr5 .req X11\n" + "c_ptr6 .req X12\n" + "c_ptr7 .req X13\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "add a_ptr4, a_ptr3, %[lda]\n" + "add c_ptr4, c_ptr3, %[ldc]\n" + "add a_ptr5, a_ptr4, %[lda]\n" + "add c_ptr5, c_ptr4, %[ldc]\n" + "add a_ptr6, a_ptr5, %[lda]\n" + "add c_ptr6, c_ptr5, %[ldc]\n" + "add a_ptr7, a_ptr6, %[lda]\n" + "add c_ptr7, c_ptr6, %[ldc]\n" + "cbz %[oob_rows], 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr7, %[c_ptr0], #0x0\n" + "add a_ptr7, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr6, %[c_ptr0], #0x0\n" + "add a_ptr6, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr5, %[c_ptr0], #0x0\n" + "add a_ptr5, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr4, %[c_ptr0], #0x0\n" + "add a_ptr4, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr3, %[c_ptr0], #0x0\n" + "add a_ptr3, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr2, %[c_ptr0], #0x0\n" + "add a_ptr2, %[a_ptr0], #0x0\n" + "b.eq 1f\n" + "subs %[oob_rows], %[oob_rows], #0x1\n" + "add c_ptr1, %[c_ptr0], #0x0\n" + "add a_ptr1, %[a_ptr0], #0x0\n" + "1:\n" + "ldr q0, [%[a_ptr0]], #0x10\n" + "ldr q2, [a_ptr1], #0x10\n" + "ldr q4, [a_ptr2], #0x10\n" + "ldr q6, [a_ptr3], #0x10\n" + "ldr q8, [a_ptr4], #0x10\n" + "ldr q10, [a_ptr5], #0x10\n" + "ldr q12, [a_ptr6], #0x10\n" + "ldr q14, [a_ptr7], #0x10\n" + "ldr q1, [%[a_ptr0]]\n" + "ldr q3, [a_ptr1]\n" + "ldr q5, [a_ptr2]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" + "ldr q7, [a_ptr3]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" + "ldr q9, [a_ptr4]\n" + "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" + "ldr q11, [a_ptr5]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" + "ldr q13, [a_ptr6]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" + "ldr q15, [a_ptr7]\n" + "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "ldr q23, [%[b_ptr0], #0x70]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "subs %[loops], %[loops], #0x1\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "fmla v24.4s, v23.4s, v1.s[3]\n" + "fmla v25.4s, v23.4s, v3.s[3]\n" + "fmla v26.4s, v23.4s, v5.s[3]\n" + "fmla v27.4s, v23.4s, v7.s[3]\n" + "fmla v28.4s, v23.4s, v9.s[3]\n" + "fmla v29.4s, v23.4s, v11.s[3]\n" + "fmla v30.4s, v23.4s, v13.s[3]\n" + "fmla v31.4s, v23.4s, v15.s[3]\n" + "b.eq 3f\n" + "4:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "mov v27.16b, v24.16b\n" + "ldr q23, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q29, [c_ptr5]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "fmla v24.4s, v23.4s, v1.s[3]\n" + "fmla v25.4s, v23.4s, v3.s[3]\n" + "fmla v26.4s, v23.4s, v5.s[3]\n" + "fmla v27.4s, v23.4s, v7.s[3]\n" + "fmla v28.4s, v23.4s, v9.s[3]\n" + "fmla v29.4s, v23.4s, v11.s[3]\n" + "fmla v30.4s, v23.4s, v13.s[3]\n" + "fmla v31.4s, v23.4s, v15.s[3]\n" + "b.ne 4b\n" + "3:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "ldr q16, [%[b_ptr0]]\n" + "ldr q17, [%[b_ptr0], #0x10]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "ldr q18, [%[b_ptr0], #0x20]\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "ldr q19, [%[b_ptr0], #0x30]\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "ldr q20, [%[b_ptr0], #0x40]\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "ldr q21, [%[b_ptr0], #0x50]\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "ldr q24, [%[biasptr]]\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "add c_ptr1, c_ptr1, #0x10\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "str q26, [c_ptr2]\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "add c_ptr2, c_ptr2, #0x10\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q27, [c_ptr3]\n" + "mov v25.16b, v24.16b\n" + "ldr q22, [%[b_ptr0], #0x60]\n" + "mov v26.16b, v24.16b\n" + "add c_ptr3, c_ptr3, #0x10\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q28, [c_ptr4]\n" + "mov v27.16b, v24.16b\n" + "ldr q23, [%[b_ptr0], #0x70]\n" + "mov v28.16b, v24.16b\n" + "add c_ptr4, c_ptr4, #0x10\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "str q29, [c_ptr5]\n" + "mov v29.16b, v24.16b\n" + "add c_ptr5, c_ptr5, #0x10\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "str q30, [c_ptr6]\n" + "mov v30.16b, v24.16b\n" + "add c_ptr6, c_ptr6, #0x10\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "str q31, [c_ptr7]\n" + "mov v31.16b, v24.16b\n" + "add c_ptr7, c_ptr7, #0x10\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "fmla v24.4s, v23.4s, v1.s[3]\n" + "fmla v25.4s, v23.4s, v3.s[3]\n" + "fmla v26.4s, v23.4s, v5.s[3]\n" + "fmla v27.4s, v23.4s, v7.s[3]\n" + "fmla v28.4s, v23.4s, v9.s[3]\n" + "fmla v29.4s, v23.4s, v11.s[3]\n" + "fmla v30.4s, v23.4s, v13.s[3]\n" + "fmla v31.4s, v23.4s, v15.s[3]\n" + "b 5f\n" + "2:\n" + "ldr q24, [%[biasptr]]\n" + "add %[biasptr], %[biasptr], %[biasinc]\n" + "mov v25.16b, v24.16b\n" + "mov v26.16b, v24.16b\n" + "mov v27.16b, v24.16b\n" + "mov v28.16b, v24.16b\n" + "mov v29.16b, v24.16b\n" + "mov v30.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v2.s[0]\n" + "fmla v26.4s, v16.4s, v4.s[0]\n" + "fmla v27.4s, v16.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v8.s[0]\n" + "fmla v29.4s, v16.4s, v10.s[0]\n" + "fmla v30.4s, v16.4s, v12.s[0]\n" + "fmla v31.4s, v16.4s, v14.s[0]\n" + "fmla v24.4s, v17.4s, v0.s[1]\n" + "fmla v25.4s, v17.4s, v2.s[1]\n" + "fmla v26.4s, v17.4s, v4.s[1]\n" + "fmla v27.4s, v17.4s, v6.s[1]\n" + "fmla v28.4s, v17.4s, v8.s[1]\n" + "fmla v29.4s, v17.4s, v10.s[1]\n" + "fmla v30.4s, v17.4s, v12.s[1]\n" + "fmla v31.4s, v17.4s, v14.s[1]\n" + "fmla v24.4s, v18.4s, v0.s[2]\n" + "fmla v25.4s, v18.4s, v2.s[2]\n" + "fmla v26.4s, v18.4s, v4.s[2]\n" + "fmla v27.4s, v18.4s, v6.s[2]\n" + "fmla v28.4s, v18.4s, v8.s[2]\n" + "fmla v29.4s, v18.4s, v10.s[2]\n" + "fmla v30.4s, v18.4s, v12.s[2]\n" + "fmla v31.4s, v18.4s, v14.s[2]\n" + "fmla v24.4s, v19.4s, v0.s[3]\n" + "fmla v25.4s, v19.4s, v2.s[3]\n" + "fmla v26.4s, v19.4s, v4.s[3]\n" + "fmla v27.4s, v19.4s, v6.s[3]\n" + "fmla v28.4s, v19.4s, v8.s[3]\n" + "fmla v29.4s, v19.4s, v10.s[3]\n" + "fmla v30.4s, v19.4s, v12.s[3]\n" + "fmla v31.4s, v19.4s, v14.s[3]\n" + "fmla v24.4s, v20.4s, v1.s[0]\n" + "fmla v25.4s, v20.4s, v3.s[0]\n" + "fmla v26.4s, v20.4s, v5.s[0]\n" + "fmla v27.4s, v20.4s, v7.s[0]\n" + "fmla v28.4s, v20.4s, v9.s[0]\n" + "fmla v29.4s, v20.4s, v11.s[0]\n" + "fmla v30.4s, v20.4s, v13.s[0]\n" + "fmla v31.4s, v20.4s, v15.s[0]\n" + "fmla v24.4s, v21.4s, v1.s[1]\n" + "fmla v25.4s, v21.4s, v3.s[1]\n" + "fmla v26.4s, v21.4s, v5.s[1]\n" + "fmla v27.4s, v21.4s, v7.s[1]\n" + "fmla v28.4s, v21.4s, v9.s[1]\n" + "fmla v29.4s, v21.4s, v11.s[1]\n" + "fmla v30.4s, v21.4s, v13.s[1]\n" + "fmla v31.4s, v21.4s, v15.s[1]\n" + "fmla v24.4s, v22.4s, v1.s[2]\n" + "fmla v25.4s, v22.4s, v3.s[2]\n" + "fmla v26.4s, v22.4s, v5.s[2]\n" + "fmla v27.4s, v22.4s, v7.s[2]\n" + "fmla v28.4s, v22.4s, v9.s[2]\n" + "fmla v29.4s, v22.4s, v11.s[2]\n" + "fmla v30.4s, v22.4s, v13.s[2]\n" + "fmla v31.4s, v22.4s, v15.s[2]\n" + "fmla v24.4s, v23.4s, v1.s[3]\n" + "fmla v25.4s, v23.4s, v3.s[3]\n" + "fmla v26.4s, v23.4s, v5.s[3]\n" + "fmla v27.4s, v23.4s, v7.s[3]\n" + "fmla v28.4s, v23.4s, v9.s[3]\n" + "fmla v29.4s, v23.4s, v11.s[3]\n" + "fmla v30.4s, v23.4s, v13.s[3]\n" + "fmla v31.4s, v23.4s, v15.s[3]\n" + "5:\n" + "ld1r {v22.4s}, [%[minptr]]\n" + "ld1r {v23.4s}, [%[maxptr]]\n" + "fmax v24.4s, v24.4s, v22.4s\n" + "fmax v25.4s, v25.4s, v22.4s\n" + "fmax v26.4s, v26.4s, v22.4s\n" + "fmax v27.4s, v27.4s, v22.4s\n" + "fmin v24.4s, v24.4s, v23.4s\n" + "fmin v25.4s, v25.4s, v23.4s\n" + "fmin v26.4s, v26.4s, v23.4s\n" + "fmin v27.4s, v27.4s, v23.4s\n" + "str q24, [%[c_ptr0]]\n" + "fmax v28.4s, v28.4s, v22.4s\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "fmax v29.4s, v29.4s, v22.4s\n" + "str q25, [c_ptr1]\n" + "fmax v30.4s, v30.4s, v22.4s\n" + "fmin v28.4s, v28.4s, v23.4s\n" + "fmax v31.4s, v31.4s, v22.4s\n" + "str q26, [c_ptr2]\n" + "fmin v29.4s, v29.4s, v23.4s\n" + "fmin v30.4s, v30.4s, v23.4s\n" + "fmin v31.4s, v31.4s, v23.4s\n" + "str q27, [c_ptr3]\n" + "str q28, [c_ptr4]\n" + "str q29, [c_ptr5]\n" + "str q30, [c_ptr6]\n" + "str q31, [c_ptr7]\n" + ".unreq a_ptr1\n" + ".unreq a_ptr2\n" + ".unreq a_ptr3\n" + ".unreq a_ptr4\n" + ".unreq a_ptr5\n" + ".unreq a_ptr6\n" + ".unreq a_ptr7\n" + ".unreq c_ptr1\n" + ".unreq c_ptr2\n" + ".unreq c_ptr3\n" + ".unreq c_ptr4\n" + ".unreq c_ptr5\n" + ".unreq c_ptr6\n" + ".unreq c_ptr7\n" + : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [oob_rows] "+r" (oob_rows), [biasptr] "+r" (biasptr) + : [lda] "r" (ldab), [ldc] "r" (ldcb), [biasinc] "r" (biasinc), [minptr] "r" (minptr), [maxptr] "r" (maxptr) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory" + ); + break; + } + } +} + +} // namespace arm_gemm + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp index 7fa02e326e..a1a1fc51af 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); -void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); +void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class smallK_hybrid_s8s32_dot_4x6 { @@ -40,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 6; } @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 6, 4, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp index e8ac2d7489..6d71840801 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp @@ -25,15 +25,16 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -97,335 +98,356 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "ins v24.d[1], temploadreg0\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -493,364 +515,385 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" + "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ins v22.d[1], temploadreg2\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v23.d[1], temploadreg3\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -918,390 +961,408 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ins v23.d[1], temploadreg3\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1369,419 +1430,437 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1914,104 +1993,104 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2023,265 +2102,283 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2420,111 +2517,111 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2535,282 +2632,300 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2949,119 +3064,119 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3073,299 +3188,317 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -3511,126 +3644,126 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3641,316 +3774,334 @@ void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp index 64ab6f0fdc..d4082130ad 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp @@ -25,15 +25,16 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -93,313 +94,322 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -459,340 +469,349 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "subs %[loops], %[loops], #0x1\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "subs %[loops], %[loops], #0x1\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -852,364 +871,370 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1269,391 +1294,397 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" - "subs %[loops], %[loops], #0x1\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "subs %[loops], %[loops], #0x1\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1778,344 +1809,362 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr q25, [%[b_ptr0], #0x10]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2246,365 +2295,383 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2735,389 +2802,407 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr q25, [%[b_ptr0], #0x10]\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "movi v26.4s, #0\n" - "str q27, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -3255,410 +3340,428 @@ void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "b.eq 7f\n" "8:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q27, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - "str q29, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "b.ne 8b\n" "7:\n" - "str q26, [%[c_ptr0]], #0x10\n" + "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" "ldr q25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q27, [c_ptr1], #0x10\n" + "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" - "str q28, [c_ptr2], #0x10\n" + ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n" + "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - ".word 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" - ".word 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" - "str q29, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n" + "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - ".word 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" - ".word 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" - "str q30, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n" + "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - ".word 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" - "str q31, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n" + "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - ".word 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q28, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q29, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q30, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q31, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp index 6615f1f36b..02be0bd77a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); -void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); +void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class smallK_hybrid_s8s32_dot_4x8 { @@ -40,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 8; } @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 8, 4, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp index 4ed2bc823c..c742fcc890 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp @@ -25,15 +25,16 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -114,47 +115,47 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -172,94 +173,118 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "ins v16.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" - "ins v16.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "subs %[loops], %[loops], #0x1\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr d16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "ins v16.d[1], temploadreg0\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr d16, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" - "ins v16.d[1], temploadreg0\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -348,55 +373,55 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -414,23 +439,23 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -441,98 +466,122 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v16.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v17.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v17.d[1], temploadreg1\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -620,56 +669,56 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -687,32 +736,32 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -726,117 +775,141 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v17.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v18.d[1], temploadreg2\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - "str q26, [c_ptr2], #0x10\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "ldr temploadreg2, [%[b_ptr0], #0x28]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v18.d[1], temploadreg2\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -925,63 +998,63 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -999,41 +1072,41 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1050,136 +1123,160 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v18.d[1], temploadreg2\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v19.d[1], temploadreg3\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v19.d[1], temploadreg3\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1334,50 +1431,50 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1397,155 +1494,179 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v19.d[1], temploadreg3\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v20.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v20.d[1], temploadreg0\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1708,59 +1829,59 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1783,174 +1904,198 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v20.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v21.d[1], temploadreg1\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v21.d[1], temploadreg1\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2113,68 +2258,68 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2200,193 +2345,217 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v21.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v22.d[1], temploadreg2\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v22.d[1], temploadreg2\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2558,77 +2727,77 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2657,212 +2826,236 @@ void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t "ins v22.d[1], temploadreg2\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ins v23.d[1], temploadreg3\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - "str q25, [c_ptr1], #0x10\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" + "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" - "movi v26.4s, #0\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" "ins v22.d[1], temploadreg2\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "ldr d23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ins v23.d[1], temploadreg3\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp index 1eb58e1cd6..2df2c30426 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp @@ -25,15 +25,16 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - UNUSED(beta); - const long loops_count = (N / 4) - 1; +void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { + const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); const long odds_count = K % 4; @@ -110,47 +111,47 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -168,90 +169,114 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - "movi v24.4s, #0\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "movi v24.4s, #0\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -332,55 +357,55 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -398,117 +423,141 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q17, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q17, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -588,56 +637,56 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -655,144 +704,168 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "ldr q17, [%[b_ptr0], #0x10]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -873,63 +946,63 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -947,41 +1020,41 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -989,129 +1062,153 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q18, [%[b_ptr0], #0x20]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1258,50 +1355,50 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1310,146 +1407,170 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q19, [%[b_ptr0], #0x30]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1604,59 +1725,59 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1666,163 +1787,187 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q20, [%[b_ptr0], #0x40]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -1977,68 +2122,68 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2049,180 +2194,204 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q21, [%[b_ptr0], #0x50]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" @@ -2386,77 +2555,77 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2468,197 +2637,221 @@ void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, "ldr q22, [%[b_ptr0], #0x60]\n" "b.eq 7f\n" "8:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v24.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "str q25, [c_ptr1], #0x10\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + "str q25, [c_ptr1]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + "add c_ptr1, c_ptr1, #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - "str q27, [c_ptr3], #0x10\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + "ldr q16, [%[b_ptr0]]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - "ldr q16, [%[b_ptr0]]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" "ldr q22, [%[b_ptr0], #0x60]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "b.ne 8b\n" "7:\n" - "str q24, [%[c_ptr0]], #0x10\n" + "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v24.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "str q25, [c_ptr1], #0x10\n" + "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" - "str q26, [c_ptr2], #0x10\n" + ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n" + "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" - ".word 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" - "str q27, [c_ptr3], #0x10\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n" + "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" - ".word 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" - "str q28, [c_ptr4], #0x10\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n" + "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" - ".word 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" - "str q29, [c_ptr5], #0x10\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n" + "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" - ".word 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" - "str q30, [c_ptr6], #0x10\n" + "add c_ptr5, c_ptr5, #0x10\n" + ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n" + "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" - ".word 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" - "str q31, [c_ptr7], #0x10\n" + "add c_ptr6, c_ptr6, #0x10\n" + ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n" + "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" - ".word 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" + "add c_ptr7, c_ptr7, #0x10\n" + ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q25, [c_ptr1]\n" + "add c_ptr1, c_ptr1, #0x10\n" "str q26, [c_ptr2]\n" + "add c_ptr2, c_ptr2, #0x10\n" "str q27, [c_ptr3]\n" + "add c_ptr3, c_ptr3, #0x10\n" "str q28, [c_ptr4]\n" + "add c_ptr4, c_ptr4, #0x10\n" "str q29, [c_ptr5]\n" + "add c_ptr5, c_ptr5, #0x10\n" "str q30, [c_ptr6]\n" + "add c_ptr6, c_ptr6, #0x10\n" "str q31, [c_ptr7]\n" + "add c_ptr7, c_ptr7, #0x10\n" ".unreq a_ptr1\n" ".unreq a_ptr2\n" ".unreq a_ptr3\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp index adeebf9bbc..1110924e5c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); -void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); +void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class smallK_hybrid_u8u32_dot_4x6 { @@ -40,7 +40,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 6, 4, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp index 8bf5debc37..ceb4a3b2d3 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp @@ -25,14 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -97,140 +98,155 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" @@ -238,114 +254,111 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - "ins v24.d[1], temploadreg0\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -354,83 +367,74 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -511,369 +515,372 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" + "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ins v22.d[1], temploadreg2\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v23.d[1], temploadreg3\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v26.4s, #0\n" - "ins v25.d[1], temploadreg1\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -954,161 +961,175 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "ins v25.d[1], temploadreg1\n" "b.eq 7f\n" @@ -1116,132 +1137,128 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q26, [%[c_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ins v23.d[1], temploadreg3\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "ins v24.d[1], temploadreg0\n" "b.ne 8b\n" "7:\n" @@ -1251,98 +1268,88 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1423,174 +1430,188 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr d24, [%[b_ptr0]]\n" + "ldr d18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + "ins v18.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + "ins v19.d[1], temploadreg3\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + "ins v20.d[1], temploadreg0\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "ins v22.d[1], temploadreg2\n" + "ins v23.d[1], temploadreg3\n" "ins v24.d[1], temploadreg0\n" "b.eq 7f\n" "8:\n" @@ -1598,141 +1619,137 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "subs %[loops], %[loops], #0x1\n" "movi v26.4s, #0\n" "ins v25.d[1], temploadreg1\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" - "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + "add c_ptr2, c_ptr2, #0x10\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" - "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add c_ptr3, c_ptr3, #0x10\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" - "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + "add c_ptr4, c_ptr4, #0x10\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" - "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + "ins v21.d[1], temploadreg1\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + "ldr temploadreg2, [%[b_ptr0], #0x8]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + "ldr temploadreg3, [%[b_ptr0], #0x18]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + "ldr temploadreg0, [%[b_ptr0], #0x28]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr temploadreg1, [%[b_ptr0], #0x38]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + "ldr d22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr d23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr d24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr d25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr d18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + "ins v18.d[1], temploadreg2\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x48]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr d19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + "ins v22.d[1], temploadreg2\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + "ins v19.d[1], temploadreg3\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + "ldr temploadreg3, [%[b_ptr0], #0x58]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr d20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + "ins v23.d[1], temploadreg3\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + "ins v20.d[1], temploadreg0\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + "ldr temploadreg0, [%[b_ptr0], #0x68]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" + "ldr d21, [%[b_ptr0], #0x30]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr d25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "ins v21.d[1], temploadreg1\n" + "ldr temploadreg1, [%[b_ptr0], #0x78]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -1742,105 +1759,95 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1986,104 +1993,104 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2105,142 +2112,142 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" @@ -2252,113 +2259,113 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2510,111 +2517,111 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2635,152 +2642,152 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" @@ -2792,120 +2799,120 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -3057,119 +3064,119 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3191,161 +3198,161 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "ins v24.d[1], temploadreg0\n" @@ -3357,128 +3364,128 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -3637,126 +3644,126 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr d24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3777,171 +3784,171 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr d25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" "ins v25.d[1], temploadreg1\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "ldr d24, [%[b_ptr0]]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" "ins v24.d[1], temploadreg0\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "ldr d25, [%[b_ptr0], #0x10]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" "b.ne 8b\n" @@ -3953,135 +3960,135 @@ void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_ "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp index 1f279afd43..4eecee4c07 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp @@ -25,14 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -93,318 +94,309 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr s2, [%[a_ptr0]]\n" - "ldr s6, [a_ptr1]\n" - "ldr s10, [a_ptr2]\n" - "ldr s14, [a_ptr3]\n" - "ldr s18, [a_ptr4]\n" - "ldr s22, [a_ptr5]\n" + "ldr s5, [a_ptr1]\n" + "ldr s8, [a_ptr2]\n" + "ldr s11, [a_ptr3]\n" + "ldr s14, [a_ptr4]\n" + "ldr s17, [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b2, [%[a_ptr0]]\n" - "ldr b6, [a_ptr1]\n" - "ldr b10, [a_ptr2]\n" - "ldr b14, [a_ptr3]\n" - "ldr b18, [a_ptr4]\n" - "ldr b22, [a_ptr5]\n" + "ldr b5, [a_ptr1]\n" + "ldr b8, [a_ptr2]\n" + "ldr b11, [a_ptr3]\n" + "ldr b14, [a_ptr4]\n" + "ldr b17, [a_ptr5]\n" "b 3f\n" "4:\n" "ldr h2, [%[a_ptr0]], #0x2\n" - "ldr h6, [a_ptr1], #0x2\n" - "ldr h10, [a_ptr2], #0x2\n" - "ldr h14, [a_ptr3], #0x2\n" - "ldr h18, [a_ptr4], #0x2\n" - "ldr h22, [a_ptr5], #0x2\n" + "ldr h5, [a_ptr1], #0x2\n" + "ldr h8, [a_ptr2], #0x2\n" + "ldr h11, [a_ptr3], #0x2\n" + "ldr h14, [a_ptr4], #0x2\n" + "ldr h17, [a_ptr5], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[2], [%[a_ptr0]]\n" - "ld1 {v6.b}[2], [a_ptr1]\n" - "ld1 {v10.b}[2], [a_ptr2]\n" - "ld1 {v14.b}[2], [a_ptr3]\n" - "ld1 {v18.b}[2], [a_ptr4]\n" - "ld1 {v22.b}[2], [a_ptr5]\n" + "ld1 {v5.b}[2], [a_ptr1]\n" + "ld1 {v8.b}[2], [a_ptr2]\n" + "ld1 {v11.b}[2], [a_ptr3]\n" + "ld1 {v14.b}[2], [a_ptr4]\n" + "ld1 {v17.b}[2], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x10\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -477,345 +469,336 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr d2, [%[a_ptr0]]\n" - "ldr d6, [a_ptr1]\n" - "ldr d10, [a_ptr2]\n" - "ldr d14, [a_ptr3]\n" - "ldr d18, [a_ptr4]\n" - "ldr d22, [a_ptr5]\n" + "ldr d5, [a_ptr1]\n" + "ldr d8, [a_ptr2]\n" + "ldr d11, [a_ptr3]\n" + "ldr d14, [a_ptr4]\n" + "ldr d17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr s2, [%[a_ptr0]], #0x4\n" - "ldr s6, [a_ptr1], #0x4\n" - "ldr s10, [a_ptr2], #0x4\n" - "ldr s14, [a_ptr3], #0x4\n" - "ldr s18, [a_ptr4], #0x4\n" - "ldr s22, [a_ptr5], #0x4\n" + "ldr s5, [a_ptr1], #0x4\n" + "ldr s8, [a_ptr2], #0x4\n" + "ldr s11, [a_ptr3], #0x4\n" + "ldr s14, [a_ptr4], #0x4\n" + "ldr s17, [a_ptr5], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[4], [%[a_ptr0]]\n" - "ld1 {v6.b}[4], [a_ptr1]\n" - "ld1 {v10.b}[4], [a_ptr2]\n" - "ld1 {v14.b}[4], [a_ptr3]\n" - "ld1 {v18.b}[4], [a_ptr4]\n" - "ld1 {v22.b}[4], [a_ptr5]\n" + "ld1 {v5.b}[4], [a_ptr1]\n" + "ld1 {v8.b}[4], [a_ptr2]\n" + "ld1 {v11.b}[4], [a_ptr3]\n" + "ld1 {v14.b}[4], [a_ptr4]\n" + "ld1 {v17.b}[4], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[2], [a_ptr1], #2\n" - "ld1 {v10.h}[2], [a_ptr2], #2\n" - "ld1 {v14.h}[2], [a_ptr3], #2\n" - "ld1 {v18.h}[2], [a_ptr4], #2\n" - "ld1 {v22.h}[2], [a_ptr5], #2\n" + "ld1 {v5.h}[2], [a_ptr1], #2\n" + "ld1 {v8.h}[2], [a_ptr2], #2\n" + "ld1 {v11.h}[2], [a_ptr3], #2\n" + "ld1 {v14.h}[2], [a_ptr4], #2\n" + "ld1 {v17.h}[2], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[6], [%[a_ptr0]]\n" - "ld1 {v6.b}[6], [a_ptr1]\n" - "ld1 {v10.b}[6], [a_ptr2]\n" - "ld1 {v14.b}[6], [a_ptr3]\n" - "ld1 {v18.b}[6], [a_ptr4]\n" - "ld1 {v22.b}[6], [a_ptr5]\n" + "ld1 {v5.b}[6], [a_ptr1]\n" + "ld1 {v8.b}[6], [a_ptr2]\n" + "ld1 {v11.b}[6], [a_ptr3]\n" + "ld1 {v14.b}[6], [a_ptr4]\n" + "ld1 {v17.b}[6], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], #0x10\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -888,369 +871,357 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr q21, [a_ptr5], #0x10\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr q16, [a_ptr5], #0x10\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v2.s}[2], [%[a_ptr0]]\n" - "ld1 {v6.s}[2], [a_ptr1]\n" - "ld1 {v10.s}[2], [a_ptr2]\n" - "ld1 {v14.s}[2], [a_ptr3]\n" - "ld1 {v18.s}[2], [a_ptr4]\n" - "ld1 {v22.s}[2], [a_ptr5]\n" + "ld1 {v5.s}[2], [a_ptr1]\n" + "ld1 {v8.s}[2], [a_ptr2]\n" + "ld1 {v11.s}[2], [a_ptr3]\n" + "ld1 {v14.s}[2], [a_ptr4]\n" + "ld1 {v17.s}[2], [a_ptr5]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[8], [%[a_ptr0]]\n" - "ld1 {v6.b}[8], [a_ptr1]\n" - "ld1 {v10.b}[8], [a_ptr2]\n" - "ld1 {v14.b}[8], [a_ptr3]\n" - "ld1 {v18.b}[8], [a_ptr4]\n" - "ld1 {v22.b}[8], [a_ptr5]\n" + "ld1 {v5.b}[8], [a_ptr1]\n" + "ld1 {v8.b}[8], [a_ptr2]\n" + "ld1 {v11.b}[8], [a_ptr3]\n" + "ld1 {v14.b}[8], [a_ptr4]\n" + "ld1 {v17.b}[8], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[4], [a_ptr1], #2\n" - "ld1 {v10.h}[4], [a_ptr2], #2\n" - "ld1 {v14.h}[4], [a_ptr3], #2\n" - "ld1 {v18.h}[4], [a_ptr4], #2\n" - "ld1 {v22.h}[4], [a_ptr5], #2\n" + "ld1 {v5.h}[4], [a_ptr1], #2\n" + "ld1 {v8.h}[4], [a_ptr2], #2\n" + "ld1 {v11.h}[4], [a_ptr3], #2\n" + "ld1 {v14.h}[4], [a_ptr4], #2\n" + "ld1 {v17.h}[4], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[10], [%[a_ptr0]]\n" - "ld1 {v6.b}[10], [a_ptr1]\n" - "ld1 {v10.b}[10], [a_ptr2]\n" - "ld1 {v14.b}[10], [a_ptr3]\n" - "ld1 {v18.b}[10], [a_ptr4]\n" - "ld1 {v22.b}[10], [a_ptr5]\n" + "ld1 {v5.b}[10], [a_ptr1]\n" + "ld1 {v8.b}[10], [a_ptr2]\n" + "ld1 {v11.b}[10], [a_ptr3]\n" + "ld1 {v14.b}[10], [a_ptr4]\n" + "ld1 {v17.b}[10], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + "add %[b_ptr0], %[b_ptr0], #0x30\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1323,396 +1294,384 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr q0, [%[a_ptr0]], #0x10\n" - "ldr q4, [a_ptr1], #0x10\n" - "ldr q8, [a_ptr2], #0x10\n" - "ldr q12, [a_ptr3], #0x10\n" - "ldr q16, [a_ptr4], #0x10\n" - "ldr q20, [a_ptr5], #0x10\n" + "ldr q3, [a_ptr1], #0x10\n" + "ldr q6, [a_ptr2], #0x10\n" + "ldr q9, [a_ptr3], #0x10\n" + "ldr q12, [a_ptr4], #0x10\n" + "ldr q15, [a_ptr5], #0x10\n" "ldr q1, [%[a_ptr0]], #0x10\n" - "ldr q5, [a_ptr1], #0x10\n" - "ldr q9, [a_ptr2], #0x10\n" - "ldr q13, [a_ptr3], #0x10\n" - "ldr q17, [a_ptr4], #0x10\n" - "ldr q21, [a_ptr5], #0x10\n" + "ldr q4, [a_ptr1], #0x10\n" + "ldr q7, [a_ptr2], #0x10\n" + "ldr q10, [a_ptr3], #0x10\n" + "ldr q13, [a_ptr4], #0x10\n" + "ldr q16, [a_ptr5], #0x10\n" "cbnz %[odds], 2f\n" "ldr q2, [%[a_ptr0]]\n" - "ldr q6, [a_ptr1]\n" - "ldr q10, [a_ptr2]\n" - "ldr q14, [a_ptr3]\n" - "ldr q18, [a_ptr4]\n" - "ldr q22, [a_ptr5]\n" + "ldr q5, [a_ptr1]\n" + "ldr q8, [a_ptr2]\n" + "ldr q11, [a_ptr3]\n" + "ldr q14, [a_ptr4]\n" + "ldr q17, [a_ptr5]\n" "b 3f\n" "2:\n" "ldr d2, [%[a_ptr0]], #0x8\n" - "ldr d6, [a_ptr1], #0x8\n" - "ldr d10, [a_ptr2], #0x8\n" - "ldr d14, [a_ptr3], #0x8\n" - "ldr d18, [a_ptr4], #0x8\n" - "ldr d22, [a_ptr5], #0x8\n" + "ldr d5, [a_ptr1], #0x8\n" + "ldr d8, [a_ptr2], #0x8\n" + "ldr d11, [a_ptr3], #0x8\n" + "ldr d14, [a_ptr4], #0x8\n" + "ldr d17, [a_ptr5], #0x8\n" "ld1 {v2.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v6.s}[2], [a_ptr1], #4\n" - "ld1 {v10.s}[2], [a_ptr2], #4\n" - "ld1 {v14.s}[2], [a_ptr3], #4\n" - "ld1 {v18.s}[2], [a_ptr4], #4\n" - "ld1 {v22.s}[2], [a_ptr5], #4\n" + "ld1 {v5.s}[2], [a_ptr1], #4\n" + "ld1 {v8.s}[2], [a_ptr2], #4\n" + "ld1 {v11.s}[2], [a_ptr3], #4\n" + "ld1 {v14.s}[2], [a_ptr4], #4\n" + "ld1 {v17.s}[2], [a_ptr5], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v2.b}[12], [%[a_ptr0]]\n" - "ld1 {v6.b}[12], [a_ptr1]\n" - "ld1 {v10.b}[12], [a_ptr2]\n" - "ld1 {v14.b}[12], [a_ptr3]\n" - "ld1 {v18.b}[12], [a_ptr4]\n" - "ld1 {v22.b}[12], [a_ptr5]\n" + "ld1 {v5.b}[12], [a_ptr1]\n" + "ld1 {v8.b}[12], [a_ptr2]\n" + "ld1 {v11.b}[12], [a_ptr3]\n" + "ld1 {v14.b}[12], [a_ptr4]\n" + "ld1 {v17.b}[12], [a_ptr5]\n" "b 3f\n" "4:\n" "ld1 {v2.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v6.h}[6], [a_ptr1], #2\n" - "ld1 {v10.h}[6], [a_ptr2], #2\n" - "ld1 {v14.h}[6], [a_ptr3], #2\n" - "ld1 {v18.h}[6], [a_ptr4], #2\n" - "ld1 {v22.h}[6], [a_ptr5], #2\n" + "ld1 {v5.h}[6], [a_ptr1], #2\n" + "ld1 {v8.h}[6], [a_ptr2], #2\n" + "ld1 {v11.h}[6], [a_ptr3], #2\n" + "ld1 {v14.h}[6], [a_ptr4], #2\n" + "ld1 {v17.h}[6], [a_ptr5], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v2.b}[14], [%[a_ptr0]]\n" - "ld1 {v6.b}[14], [a_ptr1]\n" - "ld1 {v10.b}[14], [a_ptr2]\n" - "ld1 {v14.b}[14], [a_ptr3]\n" - "ld1 {v18.b}[14], [a_ptr4]\n" - "ld1 {v22.b}[14], [a_ptr5]\n" + "ld1 {v5.b}[14], [a_ptr1]\n" + "ld1 {v8.b}[14], [a_ptr2]\n" + "ld1 {v11.b}[14], [a_ptr3]\n" + "ld1 {v14.b}[14], [a_ptr4]\n" + "ld1 {v17.b}[14], [a_ptr5]\n" "3:\n" "movi v26.4s, #0\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "movi v27.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" "movi v28.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" "movi v29.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "movi v30.4s, #0\n" - "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" "movi v31.4s, #0\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x40]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0x80]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "cbz %[loops], 6f\n" - "ldr q24, [%[b_ptr0]]\n" + "ldr q18, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" "b.eq 7f\n" "8:\n" "str q26, [%[c_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "subs %[loops], %[loops], #0x1\n" "str q27, [c_ptr1]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + "ldr q22, [%[b_ptr0], #0x40]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + "ldr q23, [%[b_ptr0], #0x50]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + "ldr q24, [%[b_ptr0], #0x60]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x70]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], #0x10\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" "movi v26.4s, #0\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" + "add %[c_ptr0], %[c_ptr0], #0x10\n" "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n" + ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n" + ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n" + "ldr q18, [%[b_ptr0]]\n" + ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n" + ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n" + ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n" + ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n" + ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n" + "ldr q19, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n" + ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n" + ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n" + ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n" + ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n" + "ldr q20, [%[b_ptr0], #0x20]\n" + ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n" + ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n" + ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n" + ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n" + ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n" + "ldr q21, [%[b_ptr0], #0x30]\n" + ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x40\n" + ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n" + ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n" + ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n" + ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n" + ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n" + ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n" + ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n" + ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n" + ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n" + ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n" + ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n" + ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n" + ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n" + ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n" + ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n" + ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n" + ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n" + ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n" + ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n" + ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n" + ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n" + ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n" + ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n" + ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n" + ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n" + ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n" + ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n" + ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n" + ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n" + ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n" + ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n" + ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n" + ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1850,104 +1809,104 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1962,120 +1921,120 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" @@ -2086,113 +2045,113 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2336,111 +2295,111 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2455,127 +2414,127 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -2586,120 +2545,120 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2843,119 +2802,119 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2970,135 +2929,135 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "ldr q24, [%[b_ptr0]]\n" "b.ne 8b\n" "7:\n" @@ -3109,128 +3068,128 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -3381,126 +3340,126 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr5, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr5, #0x100]\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x140]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr5, #0x180]\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "cbz %[loops], 6f\n" "ldr q24, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -3515,142 +3474,142 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v27.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "b.ne 8b\n" "7:\n" "str q26, [%[c_ptr0]]\n" @@ -3661,135 +3620,135 @@ void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B "str q27, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v27.4s, #0\n" - ".word 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n" "str q28, [c_ptr2]\n" "movi v28.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" + ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n" "str q29, [c_ptr3]\n" "movi v29.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" + ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n" "str q30, [c_ptr4]\n" "movi v30.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" + ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n" "str q31, [c_ptr5]\n" "movi v31.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" - ".word 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" - ".word 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" - ".word 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" - ".word 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" - ".word 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" - ".word 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" - ".word 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" - ".word 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" - ".word 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" - ".word 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" - ".word 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" - ".word 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" - ".word 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" - ".word 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" - ".word 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" - ".word 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" - ".word 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" - ".word 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" - ".word 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" - ".word 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" - ".word 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" - ".word 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" - ".word 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" - ".word 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" - ".word 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" - ".word 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" - ".word 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" - ".word 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" - ".word 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" - ".word 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" - ".word 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" - ".word 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" - ".word 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" - ".word 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" - ".word 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" - ".word 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" - ".word 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" - ".word 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" - ".word 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" - ".word 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" - ".word 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" - ".word 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" - ".word 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" - ".word 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" - ".word 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" - ".word 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" - ".word 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" - ".word 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" - ".word 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" - ".word 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" - ".word 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" - ".word 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" - ".word 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" - ".word 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" - ".word 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" - "ldr q24, [%[b_ptr0]]\n" - ".word 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" - ".word 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" - ".word 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" - ".word 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" - ".word 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" - ".word 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" - "ldr q25, [%[b_ptr0], #0x10]\n" - ".word 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" - "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" - ".word 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" - ".word 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" - ".word 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" - ".word 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" - ".word 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" - ".word 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" - ".word 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" - ".word 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" - ".word 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" - ".word 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" + ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n" + ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n" + ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n" + ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n" + ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n" + ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n" + ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n" + ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n" + ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n" + ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n" + ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n" + ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n" + ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n" + ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n" + ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n" + ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n" + ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n" + ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n" + ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n" + ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n" + ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n" + ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n" + ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n" + ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n" + ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n" + ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n" + ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n" + ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n" + ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n" + ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n" + ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n" + ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n" + ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n" + ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n" + ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n" + ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n" + ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n" + ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n" + ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n" + ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n" + ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n" + ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n" + ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n" + ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n" + ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n" + ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n" + ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n" + ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n" + ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n" + ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n" + "ldr q24, [%[b_ptr0]]\n" + ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n" + ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n" + ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n" + ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n" + ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n" + ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n" + "ldr q25, [%[b_ptr0], #0x10]\n" + ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n" + "add %[b_ptr0], %[b_ptr0], #0x20\n" + ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n" + ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n" + ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n" + ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n" + ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n" + ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n" + ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n" + ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n" + ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n" + ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n" "6:\n" "str q26, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp index a260917d08..fef8909d4f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,8 @@ namespace arm_gemm { // Actual kernel implementations -void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); -void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); +void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class smallK_hybrid_u8u32_dot_4x8 { @@ -40,7 +40,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -53,11 +53,26 @@ public: return 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsFixed<operand_type, result_type, 8, 4, 4> transforms = {}; // Default to the generic kernel diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp index ec29698f48..f30f7e2895 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp @@ -25,14 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -114,47 +115,47 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -172,14 +173,14 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -197,33 +198,33 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" @@ -241,32 +242,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -372,55 +373,55 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -438,23 +439,23 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -475,49 +476,49 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" @@ -530,40 +531,40 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -668,56 +669,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -735,32 +736,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -784,61 +785,61 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" "b.ne 8b\n" @@ -850,48 +851,48 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -997,63 +998,63 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -1071,41 +1072,41 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1132,72 +1133,72 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" "b.ne 8b\n" @@ -1209,56 +1210,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1430,50 +1431,50 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1503,83 +1504,83 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" "b.ne 8b\n" @@ -1591,64 +1592,64 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1828,59 +1829,59 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1913,94 +1914,94 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" "b.ne 8b\n" @@ -2012,72 +2013,72 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2257,68 +2258,68 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2354,105 +2355,105 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" "b.ne 8b\n" @@ -2464,80 +2465,80 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2726,77 +2727,77 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr d16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2835,116 +2836,116 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "ldr temploadreg1, [%[b_ptr0], #0x18]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" "ldr temploadreg3, [%[b_ptr0], #0x38]\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr d16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "ins v16.d[1], temploadreg0\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr d17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" "ins v17.d[1], temploadreg1\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" "ldr temploadreg1, [%[b_ptr0], #0x58]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr d18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" "ins v18.d[1], temploadreg2\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr d19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" "ins v19.d[1], temploadreg3\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" "ldr temploadreg3, [%[b_ptr0], #0x78]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr d20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" "ins v20.d[1], temploadreg0\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr d21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" "ins v21.d[1], temploadreg1\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "ldr d22, [%[b_ptr0], #0x60]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" "ins v22.d[1], temploadreg2\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "ldr d23, [%[b_ptr0], #0x70]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" "b.ne 8b\n" @@ -2956,88 +2957,88 @@ void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_ "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp index 46ca013270..16c1d1740f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp @@ -25,14 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - UNUSED(beta); +void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)4) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); @@ -110,47 +111,47 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "1:\n" "cbnz %[odds], 2f\n" "ldr s0, [%[a_ptr0]]\n" - "ldr s2, [a_ptr1]\n" - "ldr s4, [a_ptr2]\n" - "ldr s6, [a_ptr3]\n" - "ldr s8, [a_ptr4]\n" - "ldr s10, [a_ptr5]\n" - "ldr s12, [a_ptr6]\n" - "ldr s14, [a_ptr7]\n" + "ldr s1, [a_ptr1]\n" + "ldr s2, [a_ptr2]\n" + "ldr s3, [a_ptr3]\n" + "ldr s4, [a_ptr4]\n" + "ldr s5, [a_ptr5]\n" + "ldr s6, [a_ptr6]\n" + "ldr s7, [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ldr b0, [%[a_ptr0]]\n" - "ldr b2, [a_ptr1]\n" - "ldr b4, [a_ptr2]\n" - "ldr b6, [a_ptr3]\n" - "ldr b8, [a_ptr4]\n" - "ldr b10, [a_ptr5]\n" - "ldr b12, [a_ptr6]\n" - "ldr b14, [a_ptr7]\n" + "ldr b1, [a_ptr1]\n" + "ldr b2, [a_ptr2]\n" + "ldr b3, [a_ptr3]\n" + "ldr b4, [a_ptr4]\n" + "ldr b5, [a_ptr5]\n" + "ldr b6, [a_ptr6]\n" + "ldr b7, [a_ptr7]\n" "b 3f\n" "4:\n" "ldr h0, [%[a_ptr0]], #0x2\n" - "ldr h2, [a_ptr1], #0x2\n" - "ldr h4, [a_ptr2], #0x2\n" - "ldr h6, [a_ptr3], #0x2\n" - "ldr h8, [a_ptr4], #0x2\n" - "ldr h10, [a_ptr5], #0x2\n" - "ldr h12, [a_ptr6], #0x2\n" - "ldr h14, [a_ptr7], #0x2\n" + "ldr h1, [a_ptr1], #0x2\n" + "ldr h2, [a_ptr2], #0x2\n" + "ldr h3, [a_ptr3], #0x2\n" + "ldr h4, [a_ptr4], #0x2\n" + "ldr h5, [a_ptr5], #0x2\n" + "ldr h6, [a_ptr6], #0x2\n" + "ldr h7, [a_ptr7], #0x2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "ld1 {v2.b}[2], [a_ptr1]\n" - "ld1 {v4.b}[2], [a_ptr2]\n" - "ld1 {v6.b}[2], [a_ptr3]\n" - "ld1 {v8.b}[2], [a_ptr4]\n" - "ld1 {v10.b}[2], [a_ptr5]\n" - "ld1 {v12.b}[2], [a_ptr6]\n" - "ld1 {v14.b}[2], [a_ptr7]\n" + "ld1 {v1.b}[2], [a_ptr1]\n" + "ld1 {v2.b}[2], [a_ptr2]\n" + "ld1 {v3.b}[2], [a_ptr3]\n" + "ld1 {v4.b}[2], [a_ptr4]\n" + "ld1 {v5.b}[2], [a_ptr5]\n" + "ld1 {v6.b}[2], [a_ptr6]\n" + "ld1 {v7.b}[2], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -168,14 +169,14 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" "movi v31.4s, #0\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -190,33 +191,33 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" "add %[b_ptr0], %[b_ptr0], #0x10\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" @@ -233,32 +234,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -356,55 +357,55 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "1:\n" "cbnz %[odds], 2f\n" "ldr d0, [%[a_ptr0]]\n" - "ldr d2, [a_ptr1]\n" - "ldr d4, [a_ptr2]\n" - "ldr d6, [a_ptr3]\n" - "ldr d8, [a_ptr4]\n" - "ldr d10, [a_ptr5]\n" - "ldr d12, [a_ptr6]\n" - "ldr d14, [a_ptr7]\n" + "ldr d1, [a_ptr1]\n" + "ldr d2, [a_ptr2]\n" + "ldr d3, [a_ptr3]\n" + "ldr d4, [a_ptr4]\n" + "ldr d5, [a_ptr5]\n" + "ldr d6, [a_ptr6]\n" + "ldr d7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr s0, [%[a_ptr0]], #0x4\n" - "ldr s2, [a_ptr1], #0x4\n" - "ldr s4, [a_ptr2], #0x4\n" - "ldr s6, [a_ptr3], #0x4\n" - "ldr s8, [a_ptr4], #0x4\n" - "ldr s10, [a_ptr5], #0x4\n" - "ldr s12, [a_ptr6], #0x4\n" - "ldr s14, [a_ptr7], #0x4\n" + "ldr s1, [a_ptr1], #0x4\n" + "ldr s2, [a_ptr2], #0x4\n" + "ldr s3, [a_ptr3], #0x4\n" + "ldr s4, [a_ptr4], #0x4\n" + "ldr s5, [a_ptr5], #0x4\n" + "ldr s6, [a_ptr6], #0x4\n" + "ldr s7, [a_ptr7], #0x4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[4], [%[a_ptr0]]\n" - "ld1 {v2.b}[4], [a_ptr1]\n" - "ld1 {v4.b}[4], [a_ptr2]\n" - "ld1 {v6.b}[4], [a_ptr3]\n" - "ld1 {v8.b}[4], [a_ptr4]\n" - "ld1 {v10.b}[4], [a_ptr5]\n" - "ld1 {v12.b}[4], [a_ptr6]\n" - "ld1 {v14.b}[4], [a_ptr7]\n" + "ld1 {v1.b}[4], [a_ptr1]\n" + "ld1 {v2.b}[4], [a_ptr2]\n" + "ld1 {v3.b}[4], [a_ptr3]\n" + "ld1 {v4.b}[4], [a_ptr4]\n" + "ld1 {v5.b}[4], [a_ptr5]\n" + "ld1 {v6.b}[4], [a_ptr6]\n" + "ld1 {v7.b}[4], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[2], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[2], [a_ptr1], #2\n" - "ld1 {v4.h}[2], [a_ptr2], #2\n" - "ld1 {v6.h}[2], [a_ptr3], #2\n" - "ld1 {v8.h}[2], [a_ptr4], #2\n" - "ld1 {v10.h}[2], [a_ptr5], #2\n" - "ld1 {v12.h}[2], [a_ptr6], #2\n" - "ld1 {v14.h}[2], [a_ptr7], #2\n" + "ld1 {v1.h}[2], [a_ptr1], #2\n" + "ld1 {v2.h}[2], [a_ptr2], #2\n" + "ld1 {v3.h}[2], [a_ptr3], #2\n" + "ld1 {v4.h}[2], [a_ptr4], #2\n" + "ld1 {v5.h}[2], [a_ptr5], #2\n" + "ld1 {v6.h}[2], [a_ptr6], #2\n" + "ld1 {v7.h}[2], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[6], [%[a_ptr0]]\n" - "ld1 {v2.b}[6], [a_ptr1]\n" - "ld1 {v4.b}[6], [a_ptr2]\n" - "ld1 {v6.b}[6], [a_ptr3]\n" - "ld1 {v8.b}[6], [a_ptr4]\n" - "ld1 {v10.b}[6], [a_ptr5]\n" - "ld1 {v12.b}[6], [a_ptr6]\n" - "ld1 {v14.b}[6], [a_ptr7]\n" + "ld1 {v1.b}[6], [a_ptr1]\n" + "ld1 {v2.b}[6], [a_ptr2]\n" + "ld1 {v3.b}[6], [a_ptr3]\n" + "ld1 {v4.b}[6], [a_ptr4]\n" + "ld1 {v5.b}[6], [a_ptr5]\n" + "ld1 {v6.b}[6], [a_ptr6]\n" + "ld1 {v7.b}[6], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -422,23 +423,23 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x20\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -453,49 +454,49 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -506,40 +507,40 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -636,56 +637,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add a_ptr1, %[a_ptr0], #0x0\n" "1:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "cbnz %[odds], 2f\n" "ld1 {v0.s}[2], [%[a_ptr0]]\n" - "ld1 {v2.s}[2], [a_ptr1]\n" - "ld1 {v4.s}[2], [a_ptr2]\n" - "ld1 {v6.s}[2], [a_ptr3]\n" - "ld1 {v8.s}[2], [a_ptr4]\n" - "ld1 {v10.s}[2], [a_ptr5]\n" - "ld1 {v12.s}[2], [a_ptr6]\n" - "ld1 {v14.s}[2], [a_ptr7]\n" + "ld1 {v1.s}[2], [a_ptr1]\n" + "ld1 {v2.s}[2], [a_ptr2]\n" + "ld1 {v3.s}[2], [a_ptr3]\n" + "ld1 {v4.s}[2], [a_ptr4]\n" + "ld1 {v5.s}[2], [a_ptr5]\n" + "ld1 {v6.s}[2], [a_ptr6]\n" + "ld1 {v7.s}[2], [a_ptr7]\n" "b 3f\n" "2:\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[8], [%[a_ptr0]]\n" - "ld1 {v2.b}[8], [a_ptr1]\n" - "ld1 {v4.b}[8], [a_ptr2]\n" - "ld1 {v6.b}[8], [a_ptr3]\n" - "ld1 {v8.b}[8], [a_ptr4]\n" - "ld1 {v10.b}[8], [a_ptr5]\n" - "ld1 {v12.b}[8], [a_ptr6]\n" - "ld1 {v14.b}[8], [a_ptr7]\n" + "ld1 {v1.b}[8], [a_ptr1]\n" + "ld1 {v2.b}[8], [a_ptr2]\n" + "ld1 {v3.b}[8], [a_ptr3]\n" + "ld1 {v4.b}[8], [a_ptr4]\n" + "ld1 {v5.b}[8], [a_ptr5]\n" + "ld1 {v6.b}[8], [a_ptr6]\n" + "ld1 {v7.b}[8], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[4], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[4], [a_ptr1], #2\n" - "ld1 {v4.h}[4], [a_ptr2], #2\n" - "ld1 {v6.h}[4], [a_ptr3], #2\n" - "ld1 {v8.h}[4], [a_ptr4], #2\n" - "ld1 {v10.h}[4], [a_ptr5], #2\n" - "ld1 {v12.h}[4], [a_ptr6], #2\n" - "ld1 {v14.h}[4], [a_ptr7], #2\n" + "ld1 {v1.h}[4], [a_ptr1], #2\n" + "ld1 {v2.h}[4], [a_ptr2], #2\n" + "ld1 {v3.h}[4], [a_ptr3], #2\n" + "ld1 {v4.h}[4], [a_ptr4], #2\n" + "ld1 {v5.h}[4], [a_ptr5], #2\n" + "ld1 {v6.h}[4], [a_ptr6], #2\n" + "ld1 {v7.h}[4], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[10], [%[a_ptr0]]\n" - "ld1 {v2.b}[10], [a_ptr1]\n" - "ld1 {v4.b}[10], [a_ptr2]\n" - "ld1 {v6.b}[10], [a_ptr3]\n" - "ld1 {v8.b}[10], [a_ptr4]\n" - "ld1 {v10.b}[10], [a_ptr5]\n" - "ld1 {v12.b}[10], [a_ptr6]\n" - "ld1 {v14.b}[10], [a_ptr7]\n" + "ld1 {v1.b}[10], [a_ptr1]\n" + "ld1 {v2.b}[10], [a_ptr2]\n" + "ld1 {v3.b}[10], [a_ptr3]\n" + "ld1 {v4.b}[10], [a_ptr4]\n" + "ld1 {v5.b}[10], [a_ptr5]\n" + "ld1 {v6.b}[10], [a_ptr6]\n" + "ld1 {v7.b}[10], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -703,32 +704,32 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x30\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -744,58 +745,58 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -806,48 +807,48 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -945,63 +946,63 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "1:\n" "cbnz %[odds], 2f\n" "ldr q0, [%[a_ptr0]]\n" - "ldr q2, [a_ptr1]\n" - "ldr q4, [a_ptr2]\n" - "ldr q6, [a_ptr3]\n" - "ldr q8, [a_ptr4]\n" - "ldr q10, [a_ptr5]\n" - "ldr q12, [a_ptr6]\n" - "ldr q14, [a_ptr7]\n" + "ldr q1, [a_ptr1]\n" + "ldr q2, [a_ptr2]\n" + "ldr q3, [a_ptr3]\n" + "ldr q4, [a_ptr4]\n" + "ldr q5, [a_ptr5]\n" + "ldr q6, [a_ptr6]\n" + "ldr q7, [a_ptr7]\n" "b 3f\n" "2:\n" "ldr d0, [%[a_ptr0]], #0x8\n" - "ldr d2, [a_ptr1], #0x8\n" - "ldr d4, [a_ptr2], #0x8\n" - "ldr d6, [a_ptr3], #0x8\n" - "ldr d8, [a_ptr4], #0x8\n" - "ldr d10, [a_ptr5], #0x8\n" - "ldr d12, [a_ptr6], #0x8\n" - "ldr d14, [a_ptr7], #0x8\n" + "ldr d1, [a_ptr1], #0x8\n" + "ldr d2, [a_ptr2], #0x8\n" + "ldr d3, [a_ptr3], #0x8\n" + "ldr d4, [a_ptr4], #0x8\n" + "ldr d5, [a_ptr5], #0x8\n" + "ldr d6, [a_ptr6], #0x8\n" + "ldr d7, [a_ptr7], #0x8\n" "ld1 {v0.s}[2], [%[a_ptr0]], #4\n" - "ld1 {v2.s}[2], [a_ptr1], #4\n" - "ld1 {v4.s}[2], [a_ptr2], #4\n" - "ld1 {v6.s}[2], [a_ptr3], #4\n" - "ld1 {v8.s}[2], [a_ptr4], #4\n" - "ld1 {v10.s}[2], [a_ptr5], #4\n" - "ld1 {v12.s}[2], [a_ptr6], #4\n" - "ld1 {v14.s}[2], [a_ptr7], #4\n" + "ld1 {v1.s}[2], [a_ptr1], #4\n" + "ld1 {v2.s}[2], [a_ptr2], #4\n" + "ld1 {v3.s}[2], [a_ptr3], #4\n" + "ld1 {v4.s}[2], [a_ptr4], #4\n" + "ld1 {v5.s}[2], [a_ptr5], #4\n" + "ld1 {v6.s}[2], [a_ptr6], #4\n" + "ld1 {v7.s}[2], [a_ptr7], #4\n" "subs %[odds], %[odds], #0x1\n" "b.ne 4f\n" "ld1 {v0.b}[12], [%[a_ptr0]]\n" - "ld1 {v2.b}[12], [a_ptr1]\n" - "ld1 {v4.b}[12], [a_ptr2]\n" - "ld1 {v6.b}[12], [a_ptr3]\n" - "ld1 {v8.b}[12], [a_ptr4]\n" - "ld1 {v10.b}[12], [a_ptr5]\n" - "ld1 {v12.b}[12], [a_ptr6]\n" - "ld1 {v14.b}[12], [a_ptr7]\n" + "ld1 {v1.b}[12], [a_ptr1]\n" + "ld1 {v2.b}[12], [a_ptr2]\n" + "ld1 {v3.b}[12], [a_ptr3]\n" + "ld1 {v4.b}[12], [a_ptr4]\n" + "ld1 {v5.b}[12], [a_ptr5]\n" + "ld1 {v6.b}[12], [a_ptr6]\n" + "ld1 {v7.b}[12], [a_ptr7]\n" "b 3f\n" "4:\n" "ld1 {v0.h}[6], [%[a_ptr0]], #2\n" - "ld1 {v2.h}[6], [a_ptr1], #2\n" - "ld1 {v4.h}[6], [a_ptr2], #2\n" - "ld1 {v6.h}[6], [a_ptr3], #2\n" - "ld1 {v8.h}[6], [a_ptr4], #2\n" - "ld1 {v10.h}[6], [a_ptr5], #2\n" - "ld1 {v12.h}[6], [a_ptr6], #2\n" - "ld1 {v14.h}[6], [a_ptr7], #2\n" + "ld1 {v1.h}[6], [a_ptr1], #2\n" + "ld1 {v2.h}[6], [a_ptr2], #2\n" + "ld1 {v3.h}[6], [a_ptr3], #2\n" + "ld1 {v4.h}[6], [a_ptr4], #2\n" + "ld1 {v5.h}[6], [a_ptr5], #2\n" + "ld1 {v6.h}[6], [a_ptr6], #2\n" + "ld1 {v7.h}[6], [a_ptr7], #2\n" "subs %[odds], %[odds], #0x1\n" "b.ne 5f\n" "b 3f\n" "5:\n" "ld1 {v0.b}[14], [%[a_ptr0]]\n" - "ld1 {v2.b}[14], [a_ptr1]\n" - "ld1 {v4.b}[14], [a_ptr2]\n" - "ld1 {v6.b}[14], [a_ptr3]\n" - "ld1 {v8.b}[14], [a_ptr4]\n" - "ld1 {v10.b}[14], [a_ptr5]\n" - "ld1 {v12.b}[14], [a_ptr6]\n" - "ld1 {v14.b}[14], [a_ptr7]\n" + "ld1 {v1.b}[14], [a_ptr1]\n" + "ld1 {v2.b}[14], [a_ptr2]\n" + "ld1 {v3.b}[14], [a_ptr3]\n" + "ld1 {v4.b}[14], [a_ptr4]\n" + "ld1 {v5.b}[14], [a_ptr5]\n" + "ld1 {v6.b}[14], [a_ptr6]\n" + "ld1 {v7.b}[14], [a_ptr7]\n" "3:\n" "movi v24.4s, #0\n" "ldr q16, [%[b_ptr0]]\n" @@ -1019,41 +1020,41 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x40\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1070,67 +1071,67 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -1141,56 +1142,56 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n" + ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n" + ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n" + ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n" + ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n" + ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n" + ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n" + ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n" + ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n" + ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n" + ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1354,50 +1355,50 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x50\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1415,76 +1416,76 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -1495,64 +1496,64 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -1724,59 +1725,59 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x60\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -1795,85 +1796,85 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -1884,72 +1885,72 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2121,68 +2122,68 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x70\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2202,94 +2203,94 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -2300,80 +2301,80 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" @@ -2554,77 +2555,77 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "ldr q22, [%[b_ptr0], #0x60]\n" "movi v31.4s, #0\n" "ldr q23, [%[b_ptr0], #0x70]\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x80]\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x100]\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x140]\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "prfm PLDL1KEEP, [a_ptr7, #0x180]\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "cbz %[loops], 6f\n" "ldr q16, [%[b_ptr0]]\n" "subs %[loops], %[loops], #0x1\n" @@ -2645,103 +2646,103 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "add %[c_ptr0], %[c_ptr0], #0x10\n" "movi v25.4s, #0\n" "add c_ptr1, c_ptr1, #0x10\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" "ldr q16, [%[b_ptr0]]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" "prfm PSTL1KEEP, [c_ptr1, #0x40]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" "prfm PSTL1KEEP, [c_ptr2, #0x40]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" "prfm PSTL1KEEP, [c_ptr3, #0x40]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" "prfm PSTL1KEEP, [c_ptr4, #0x40]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" "prfm PSTL1KEEP, [c_ptr5, #0x40]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" "prfm PSTL1KEEP, [c_ptr6, #0x40]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" "prfm PSTL1KEEP, [c_ptr7, #0x40]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" "ldr q17, [%[b_ptr0], #0x10]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" "ldr q18, [%[b_ptr0], #0x20]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" "ldr q19, [%[b_ptr0], #0x30]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" "ldr q20, [%[b_ptr0], #0x40]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" "ldr q21, [%[b_ptr0], #0x50]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" "ldr q22, [%[b_ptr0], #0x60]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "b.ne 8b\n" "7:\n" "str q24, [%[c_ptr0]]\n" @@ -2752,88 +2753,88 @@ void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B "str q25, [c_ptr1]\n" "add c_ptr1, c_ptr1, #0x10\n" "movi v25.4s, #0\n" - ".word 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n" "str q26, [c_ptr2]\n" "movi v26.4s, #0\n" "add c_ptr2, c_ptr2, #0x10\n" - ".word 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" + ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n" "str q27, [c_ptr3]\n" "movi v27.4s, #0\n" "add c_ptr3, c_ptr3, #0x10\n" - ".word 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" + ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n" "str q28, [c_ptr4]\n" "movi v28.4s, #0\n" "add c_ptr4, c_ptr4, #0x10\n" - ".word 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" + ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n" "str q29, [c_ptr5]\n" "movi v29.4s, #0\n" "add c_ptr5, c_ptr5, #0x10\n" - ".word 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" + ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n" "str q30, [c_ptr6]\n" "movi v30.4s, #0\n" "add c_ptr6, c_ptr6, #0x10\n" - ".word 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" + ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n" "str q31, [c_ptr7]\n" "movi v31.4s, #0\n" "add c_ptr7, c_ptr7, #0x10\n" - ".word 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" - ".word 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" - ".word 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" - ".word 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" - ".word 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" - ".word 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" - ".word 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" - ".word 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" - ".word 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" - ".word 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" - ".word 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" - ".word 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" - ".word 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" - ".word 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" - ".word 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" - ".word 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" - ".word 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" - ".word 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" - ".word 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" - ".word 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" - ".word 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" - ".word 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" - ".word 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" - ".word 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" - ".word 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" - ".word 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" - ".word 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" - ".word 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" - ".word 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" - ".word 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" - ".word 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" - ".word 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" - ".word 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" - ".word 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" - ".word 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" - ".word 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" - ".word 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" - ".word 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" - ".word 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" - ".word 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" - ".word 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" - ".word 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" - ".word 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" - ".word 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" - ".word 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" - ".word 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" - ".word 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" - ".word 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" - ".word 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" - ".word 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" - ".word 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" - ".word 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" - ".word 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" - ".word 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" - ".word 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" - ".word 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" - ".word 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" - ".word 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" + ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n" + ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n" + ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n" + ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n" + ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n" + ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n" + ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n" + ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n" + ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n" + ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n" + ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n" + ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n" + ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n" + ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n" + ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n" + ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n" + ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n" + ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n" + ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n" + ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n" + ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n" + ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n" + ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n" + ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n" + ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n" + ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n" + ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n" + ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n" + ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n" + ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n" + ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n" + ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n" + ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n" + ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n" + ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n" + ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n" + ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n" + ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n" + ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n" + ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n" + ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n" + ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n" + ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n" + ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n" + ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n" + ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n" + ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n" + ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n" + ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n" + ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n" + ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n" + ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n" + ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n" + ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n" + ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n" "6:\n" "str q24, [%[c_ptr0]]\n" "add %[c_ptr0], %[c_ptr0], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp index c6895a6a0e..28ef8071c2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, __fp16 *, int, __fp16, int, int, int); +void sve_hybrid_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); class hybrid_fp16_mla_4VLx4 { @@ -40,10 +40,10 @@ public: typedef __fp16 operand_type; typedef __fp16 result_type; - typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, __fp16 *, int, __fp16, int, int, int); + typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length<__fp16>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + StdTransformsSVE<operand_type, result_type, 4, 4, 1> transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_fp16_mla_4VLx4; - hybrid_fp16_mla_4VLx4(const CPUInfo *ci) - { - - } + hybrid_fp16_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp index ab41fb3743..2998f33d87 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,37 +25,58 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 *C, int ldc, __fp16 beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 *C, int ldc, int M, int N, int K, const __fp16 *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 8) / 16) - 1; K -= loops_count * 16; const long regs_count = (K / 8) - 1; K -= (regs_count + 1) * 8; const long leftovers = K; + __fp16 nullbias[512]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<__fp16>() * sizeof(__fp16))); + } + __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity()); + const __fp16 * const minptr = &minval; + const __fp16 * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const __fp16 * const a_ptr0_base = A + (y * lda); const unsigned long ldab = lda * sizeof(__fp16); __fp16 *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(__fp16); for (int x0=0; x0<N; x0+=(4 * get_vector_length<__fp16>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<__fp16>())); - const __fp16 *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = leftovers; const __fp16 *a_ptr0 = a_ptr0_base; const __fp16 *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(__fp16); + const __fp16 *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -69,44 +90,37 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z18.h, #0\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.h, #0\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -322,13 +336,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[0]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" - "fmla z17.h, z13.h, z4.h[1]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z4.h[1]\n" + "fmla z17.h, z13.h, z4.h[1]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z4.h[1]\n" + "fmla z18.h, z14.h, z4.h[1]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z4.h[1]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -339,9 +352,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[2]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.h, z13.h, z4.h[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z18.h, z14.h, z4.h[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z19.h, z15.h, z4.h[3]\n" "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" @@ -372,76 +387,77 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[6]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z12.h, z4.h[7]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z17.h, z13.h, z4.h[7]\n" "fmla z18.h, z14.h, z4.h[7]\n" "fmla z19.h, z15.h, z4.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[0]\n" "fmla z18.h, z10.h, z0.h[0]\n" "fmla z19.h, z11.h, z0.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z0.h[1]\n" "fmla z18.h, z14.h, z0.h[1]\n" "fmla z19.h, z15.h, z0.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[2]\n" "fmla z18.h, z10.h, z0.h[2]\n" "fmla z19.h, z11.h, z0.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.h, z13.h, z0.h[3]\n" "fmla z18.h, z14.h, z0.h[3]\n" "fmla z19.h, z15.h, z0.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[4]\n" "fmla z18.h, z10.h, z0.h[4]\n" "fmla z19.h, z11.h, z0.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z0.h[5]\n" "fmla z18.h, z14.h, z0.h[5]\n" "fmla z19.h, z15.h, z0.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" "fmla z18.h, z10.h, z0.h[6]\n" @@ -451,33 +467,34 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[1]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z17.h, z13.h, z0.h[1]\n" + "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[1]\n" + "fmla z17.h, z13.h, z0.h[1]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[1]\n" + "fmla z18.h, z14.h, z0.h[1]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "fmla z16.h, z8.h, z0.h[2]\n" + "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "fmla z17.h, z9.h, z0.h[2]\n" + "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" "fmla z18.h, z10.h, z0.h[2]\n" "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z19.h, z11.h, z0.h[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.h, z12.h, z0.h[3]\n" - "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[3]\n" + "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[3]\n" + "fmla z18.h, z14.h, z0.h[3]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[3]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" @@ -504,88 +521,99 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[6]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z12.h, z0.h[7]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z17.h, z13.h, z0.h[7]\n" "fmla z18.h, z14.h, z0.h[7]\n" "fmla z19.h, z15.h, z0.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z4.h[0]\n" "fmla z18.h, z10.h, z4.h[0]\n" "fmla z19.h, z11.h, z4.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z4.h[1]\n" "fmla z18.h, z14.h, z4.h[1]\n" "fmla z19.h, z15.h, z4.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z4.h[2]\n" "fmla z18.h, z10.h, z4.h[2]\n" "fmla z19.h, z11.h, z4.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.h, z13.h, z4.h[3]\n" "fmla z18.h, z14.h, z4.h[3]\n" "fmla z19.h, z15.h, z4.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z4.h[4]\n" "fmla z18.h, z10.h, z4.h[4]\n" "fmla z19.h, z11.h, z4.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.h, z13.h, z4.h[5]\n" "fmla z18.h, z14.h, z4.h[5]\n" "fmla z19.h, z15.h, z4.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" "fmla z18.h, z10.h, z4.h[6]\n" "fmla z19.h, z11.h, z4.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -604,60 +632,49 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "mov z21.d, z17.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z22.d, z18.d\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z19.h, #0\n" + "mov z23.d, z19.d\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.h, #0\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.h, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.h, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z23.h, #0\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -974,9 +991,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z20.h, z8.h, z5.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.h, z9.h, z5.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z22.h, z10.h, z5.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z4.h[0]\n" @@ -1044,6 +1063,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" "fmla z21.h, z9.h, z5.h[6]\n" @@ -1060,13 +1080,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" "fmla z17.h, z9.h, z0.h[0]\n" "fmla z21.h, z9.h, z1.h[0]\n" @@ -1075,12 +1094,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[1]\n" "fmla z17.h, z13.h, z0.h[1]\n" "fmla z21.h, z13.h, z1.h[1]\n" @@ -1089,12 +1108,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[1]\n" "fmla z23.h, z15.h, z1.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[2]\n" "fmla z17.h, z9.h, z0.h[2]\n" "fmla z21.h, z9.h, z1.h[2]\n" @@ -1103,12 +1123,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[2]\n" "fmla z23.h, z11.h, z1.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z1.h[3]\n" "fmla z17.h, z13.h, z0.h[3]\n" "fmla z21.h, z13.h, z1.h[3]\n" @@ -1117,13 +1137,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[3]\n" "fmla z23.h, z15.h, z1.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[4]\n" "fmla z17.h, z9.h, z0.h[4]\n" "fmla z21.h, z9.h, z1.h[4]\n" @@ -1132,12 +1151,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z0.h[4]\n" "fmla z23.h, z11.h, z1.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[5]\n" "fmla z17.h, z13.h, z0.h[5]\n" "fmla z21.h, z13.h, z1.h[5]\n" @@ -1146,10 +1165,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[5]\n" "fmla z23.h, z15.h, z1.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" @@ -1163,19 +1183,21 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" - "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z5.h, p6/z, [a_ptr1]\n" + "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" "fmla z21.h, z9.h, z1.h[0]\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[0]\n" + "ld1rqh z5.h, p6/z, [a_ptr1]\n" "fmla z22.h, z10.h, z1.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z20.h, z12.h, z1.h[1]\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -1237,6 +1259,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" "fmla z21.h, z9.h, z1.h[6]\n" @@ -1253,13 +1276,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z0.h[7]\n" "fmla z23.h, z15.h, z1.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[0]\n" "fmla z17.h, z9.h, z4.h[0]\n" "fmla z21.h, z9.h, z5.h[0]\n" @@ -1268,12 +1290,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[0]\n" "fmla z23.h, z11.h, z5.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[1]\n" "fmla z17.h, z13.h, z4.h[1]\n" "fmla z21.h, z13.h, z5.h[1]\n" @@ -1282,12 +1304,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[1]\n" "fmla z23.h, z15.h, z5.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[2]\n" "fmla z17.h, z9.h, z4.h[2]\n" "fmla z21.h, z9.h, z5.h[2]\n" @@ -1296,12 +1319,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[2]\n" "fmla z23.h, z11.h, z5.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z5.h[3]\n" "fmla z17.h, z13.h, z4.h[3]\n" "fmla z21.h, z13.h, z5.h[3]\n" @@ -1310,13 +1333,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[3]\n" "fmla z23.h, z15.h, z5.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[4]\n" "fmla z17.h, z9.h, z4.h[4]\n" "fmla z21.h, z9.h, z5.h[4]\n" @@ -1325,12 +1347,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[4]\n" "fmla z23.h, z11.h, z5.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[5]\n" "fmla z17.h, z13.h, z4.h[5]\n" "fmla z21.h, z13.h, z5.h[5]\n" @@ -1339,10 +1361,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z15.h, z4.h[5]\n" "fmla z23.h, z15.h, z5.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" @@ -1352,9 +1375,27 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z19.h, z11.h, z4.h[6]\n" "fmla z23.h, z11.h, z5.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1h z20.h, p0, [c_ptr1]\n" @@ -1364,7 +1405,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -1387,76 +1428,61 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "mov z21.d, z17.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z22.d, z18.d\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" + "mov z23.d, z19.d\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z20.h, #0\n" + "mov z24.d, z16.d\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.h, #0\n" + "mov z25.d, z17.d\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.h, #0\n" + "mov z26.d, z18.d\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.h, #0\n" + "mov z27.d, z19.d\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z24.h, #0\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z25.h, #0\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z26.h, #0\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.h, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1872,10 +1898,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z4.h[0]\n" "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n" "fmla z20.h, z8.h, z5.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.h, z8.h, z6.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" @@ -1968,6 +1997,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" "fmla z17.h, z9.h, z4.h[6]\n" @@ -1992,13 +2022,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" "fmla z24.h, z8.h, z2.h[0]\n" "fmla z17.h, z9.h, z0.h[0]\n" @@ -2011,12 +2040,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[1]\n" "fmla z24.h, z12.h, z2.h[1]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -2029,12 +2058,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[1]\n" "fmla z27.h, z15.h, z2.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[2]\n" "fmla z24.h, z8.h, z2.h[2]\n" "fmla z17.h, z9.h, z0.h[2]\n" @@ -2047,12 +2077,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z1.h[2]\n" "fmla z27.h, z11.h, z2.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z1.h[3]\n" "fmla z24.h, z12.h, z2.h[3]\n" "fmla z17.h, z13.h, z0.h[3]\n" @@ -2065,13 +2095,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[3]\n" "fmla z27.h, z15.h, z2.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[4]\n" "fmla z24.h, z8.h, z2.h[4]\n" "fmla z17.h, z9.h, z0.h[4]\n" @@ -2084,12 +2113,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z1.h[4]\n" "fmla z27.h, z11.h, z2.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[5]\n" "fmla z24.h, z12.h, z2.h[5]\n" "fmla z17.h, z13.h, z0.h[5]\n" @@ -2102,10 +2131,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[5]\n" "fmla z27.h, z15.h, z2.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" @@ -2125,18 +2155,21 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" "fmla z24.h, z8.h, z2.h[0]\n" - "ld1rqh z5.h, p6/z, [a_ptr1]\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z6.h, p6/z, [a_ptr2]\n" + "ld1rqh z5.h, p6/z, [a_ptr1]\n" "fmla z21.h, z9.h, z1.h[0]\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "ld1rqh z6.h, p6/z, [a_ptr2]\n" "fmla z25.h, z9.h, z2.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -2222,6 +2255,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" "fmla z17.h, z9.h, z0.h[6]\n" @@ -2246,13 +2280,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z1.h[7]\n" "fmla z27.h, z15.h, z2.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[0]\n" "fmla z24.h, z8.h, z6.h[0]\n" "fmla z17.h, z9.h, z4.h[0]\n" @@ -2265,12 +2298,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[0]\n" "fmla z27.h, z11.h, z6.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[1]\n" "fmla z24.h, z12.h, z6.h[1]\n" "fmla z17.h, z13.h, z4.h[1]\n" @@ -2283,12 +2316,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[1]\n" "fmla z27.h, z15.h, z6.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[2]\n" "fmla z24.h, z8.h, z6.h[2]\n" "fmla z17.h, z9.h, z4.h[2]\n" @@ -2301,12 +2335,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[2]\n" "fmla z27.h, z11.h, z6.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z5.h[3]\n" "fmla z24.h, z12.h, z6.h[3]\n" "fmla z17.h, z13.h, z4.h[3]\n" @@ -2319,13 +2353,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[3]\n" "fmla z27.h, z15.h, z6.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[4]\n" "fmla z24.h, z8.h, z6.h[4]\n" "fmla z17.h, z9.h, z4.h[4]\n" @@ -2338,12 +2371,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[4]\n" "fmla z27.h, z11.h, z6.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[5]\n" "fmla z24.h, z12.h, z6.h[5]\n" "fmla z17.h, z13.h, z4.h[5]\n" @@ -2356,10 +2389,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z15.h, z5.h[5]\n" "fmla z27.h, z15.h, z6.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" @@ -2373,13 +2407,39 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z23.h, z11.h, z5.h[6]\n" "fmla z27.h, z11.h, z6.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" "st1h z24.h, p0, [c_ptr2]\n" @@ -2391,7 +2451,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -2419,92 +2479,73 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "whilelt p2.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" + "cbnz %[append], 1f\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "mov z21.d, z17.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z22.d, z18.d\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" + "mov z23.d, z19.d\n" "ld1rqh z3.h, p7/z, [a_ptr3]\n" - "mov z20.h, #0\n" + "mov z24.d, z16.d\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "mov z21.h, #0\n" + "mov z25.d, z17.d\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.h, #0\n" + "mov z26.d, z18.d\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.h, #0\n" + "mov z27.d, z19.d\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.h, #0\n" + "mov z28.d, z16.d\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z25.h, #0\n" + "mov z29.d, z17.d\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z26.h, #0\n" + "mov z30.d, z18.d\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z27.h, #0\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.h, #0\n" + "mov z31.d, z19.d\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.h, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.h, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.h, #0\n" "add a_ptr3, a_ptr3, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" "ld1h z16.h, p0/z, [%[c_ptr0]]\n" "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" "ld1h z28.h, p0/z, [c_ptr3]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" "ld1h z29.h, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" "ld1h z30.h, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" "ld1h z31.h, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z28.h, p7/m, z28.h, z15.h\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z29.h, p7/m, z29.h, z15.h\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z30.h, p7/m, z30.h, z15.h\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z31.h, p7/m, z31.h, z15.h\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqh z3.h, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -3021,11 +3062,15 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z16.h, z8.h, z4.h[0]\n" "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n" "fmla z20.h, z8.h, z5.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.h, z8.h, z6.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z28.h, z8.h, z7.h[0]\n" "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" "fmla z29.h, z9.h, z7.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -3141,6 +3186,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z31.h, z15.h, z7.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" "fmla z28.h, z8.h, z7.h[6]\n" @@ -3173,13 +3219,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[7]\n" "fmla z31.h, z15.h, z7.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" "fmla z24.h, z8.h, z2.h[0]\n" "fmla z28.h, z8.h, z3.h[0]\n" @@ -3196,12 +3241,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z2.h[0]\n" "fmla z31.h, z11.h, z3.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[1]\n" "fmla z24.h, z12.h, z2.h[1]\n" "fmla z28.h, z12.h, z3.h[1]\n" @@ -3218,12 +3263,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[1]\n" "fmla z31.h, z15.h, z3.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[2]\n" "fmla z24.h, z8.h, z2.h[2]\n" "fmla z28.h, z8.h, z3.h[2]\n" @@ -3240,12 +3286,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z2.h[2]\n" "fmla z31.h, z11.h, z3.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z1.h[3]\n" "fmla z24.h, z12.h, z2.h[3]\n" "fmla z28.h, z12.h, z3.h[3]\n" @@ -3262,13 +3308,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[3]\n" "fmla z31.h, z15.h, z3.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z0.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[4]\n" "fmla z24.h, z8.h, z2.h[4]\n" "fmla z28.h, z8.h, z3.h[4]\n" @@ -3285,12 +3330,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z2.h[4]\n" "fmla z31.h, z11.h, z3.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z1.h[5]\n" "fmla z24.h, z12.h, z2.h[5]\n" "fmla z28.h, z12.h, z3.h[5]\n" @@ -3307,10 +3352,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[5]\n" "fmla z31.h, z15.h, z3.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" @@ -3336,17 +3382,21 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" "fmla z28.h, z8.h, z3.h[0]\n" - "ld1rqh z6.h, p6/z, [a_ptr2]\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "ld1rqh z7.h, p6/z, [a_ptr3]\n" + "ld1rqh z6.h, p6/z, [a_ptr2]\n" "fmla z21.h, z9.h, z1.h[0]\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "ld1rqh z7.h, p6/z, [a_ptr3]\n" "fmla z25.h, z9.h, z2.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.h, z9.h, z3.h[0]\n" "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.h, z10.h, z3.h[0]\n" "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" @@ -3456,6 +3506,7 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z31.h, z15.h, z3.h[5]\n" "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.h, z8.h, z0.h[6]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.h, z8.h, z1.h[6]\n" "fmla z24.h, z8.h, z2.h[6]\n" "fmla z28.h, z8.h, z3.h[6]\n" @@ -3488,13 +3539,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z2.h[7]\n" "fmla z31.h, z15.h, z3.h[7]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[0]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[0]\n" "fmla z24.h, z8.h, z6.h[0]\n" "fmla z28.h, z8.h, z7.h[0]\n" @@ -3511,12 +3561,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[0]\n" "fmla z31.h, z11.h, z7.h[0]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[1]\n" "fmla z24.h, z12.h, z6.h[1]\n" "fmla z28.h, z12.h, z7.h[1]\n" @@ -3533,12 +3583,13 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[1]\n" "fmla z31.h, z15.h, z7.h[1]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[2]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[2]\n" "fmla z24.h, z8.h, z6.h[2]\n" "fmla z28.h, z8.h, z7.h[2]\n" @@ -3555,12 +3606,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[2]\n" "fmla z31.h, z11.h, z7.h[2]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z16.h, z12.h, z4.h[3]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.h, z12.h, z5.h[3]\n" "fmla z24.h, z12.h, z6.h[3]\n" "fmla z28.h, z12.h, z7.h[3]\n" @@ -3577,13 +3628,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[3]\n" "fmla z31.h, z15.h, z7.h[3]\n" "b.eq 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.h, z8.h, z4.h[4]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z5.h[4]\n" "fmla z24.h, z8.h, z6.h[4]\n" "fmla z28.h, z8.h, z7.h[4]\n" @@ -3600,12 +3650,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[4]\n" "fmla z31.h, z11.h, z7.h[4]\n" "b.eq 5f\n" - "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.h, z12.h, z4.h[5]\n" - "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.h, z12.h, z5.h[5]\n" "fmla z24.h, z12.h, z6.h[5]\n" "fmla z28.h, z12.h, z7.h[5]\n" @@ -3622,10 +3672,11 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z15.h, z6.h[5]\n" "fmla z31.h, z15.h, z7.h[5]\n" "b.eq 5f\n" - "ld1h z8.h, p7/z, [%[b_ptr0]]\n" - "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.h, z8.h, z4.h[6]\n" "fmla z20.h, z8.h, z5.h[6]\n" "fmla z24.h, z8.h, z6.h[6]\n" @@ -3643,16 +3694,50 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 "fmla z27.h, z11.h, z6.h[6]\n" "fmla z31.h, z11.h, z7.h[6]\n" "5:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmax z28.h, p7/m, z28.h, z14.h\n" + "fmax z29.h, p7/m, z29.h, z14.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" + "fmax z30.h, p7/m, z30.h, z14.h\n" + "fmin z28.h, p7/m, z28.h, z15.h\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.h, p7/m, z29.h, z15.h\n" + "fmax z31.h, p7/m, z31.h, z14.h\n" + "fmin z30.h, p7/m, z30.h, z15.h\n" "st1h z24.h, p0, [c_ptr2]\n" + "fmin z31.h, p7/m, z31.h, z15.h\n" "st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n" "st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n" "st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n" @@ -3667,11 +3752,12 @@ void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp index 76f452d963..8e3c17917b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, float, int, int, int); +void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); class hybrid_fp32_mla_4VLx4 { @@ -40,10 +40,10 @@ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length<float>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + StdTransformsSVE<operand_type, result_type, 4, 4, 1> transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_fp32_mla_4VLx4; - hybrid_fp32_mla_4VLx4(const CPUInfo *ci) - { - - } + hybrid_fp32_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp index 1b2d7852a8..855d27a151 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,37 +25,58 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const int K_stride = K; const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long leftovers = K; + float nullbias[256]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<float>() * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const float * const a_ptr0_base = A + (y * lda); const unsigned long ldab = lda * sizeof(float); float *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(float); for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>())); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = leftovers; const float *a_ptr0 = a_ptr0_base; const float *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(float); + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -69,269 +90,283 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.s, #0\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "b 2f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "2:\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "subs %[loops], %[loops], #0x1\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z19.s, z11.s, z0.s[2]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z16.s, z12.s, z0.s[3]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z17.s, z13.s, z0.s[3]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[2]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z19.s, z11.s, z4.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "fmla z17.s, z13.s, z4.s[3]\n" + "fmla z18.s, z14.s, z4.s[3]\n" + "fmla z19.s, z15.s, z4.s[3]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z17.s, z13.s, z4.s[3]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "fmla z18.s, z14.s, z4.s[3]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "fmla z19.s, z15.s, z4.s[3]\n" "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "fmla z19.s, z11.s, z0.s[2]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" "fmla z17.s, z13.s, z0.s[3]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z9.s, z4.s[2]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z19.s, z11.s, z4.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" + "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.s, z13.s, z4.s[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z18.s, z14.s, z4.s[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z19.s, z15.s, z4.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z19.s, z11.s, z0.s[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.s, z12.s, z0.s[3]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z18.s, z10.s, z4.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z18.s, z14.s, z4.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -350,83 +385,74 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "mov z21.d, z17.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.d, z18.d\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" + "mov z23.d, z19.d\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.s, #0\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.s, #0\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z23.s, #0\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" - "b 2f\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "2:\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "subs %[loops], %[loops], #0x1\n" "fmla z23.s, z11.s, z1.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z20.s, z12.s, z1.s[1]\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z21.s, z13.s, z1.s[1]\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" @@ -440,12 +466,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z22.s, z10.s, z1.s[2]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z22.s, z10.s, z1.s[2]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" @@ -491,12 +517,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z22.s, z10.s, z5.s[2]\n" + "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.s, z11.s, z4.s[2]\n" + "fmla z22.s, z10.s, z5.s[2]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" @@ -510,19 +536,19 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -545,12 +571,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "fmla z22.s, z10.s, z1.s[2]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z22.s, z10.s, z1.s[2]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" @@ -571,9 +597,11 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.s, z9.s, z5.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z22.s, z10.s, z5.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" @@ -592,6 +620,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" @@ -607,14 +636,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z5.s[3]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" "fmla z17.s, z9.s, z0.s[0]\n" "fmla z21.s, z9.s, z1.s[0]\n" @@ -622,13 +650,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z1.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" @@ -636,11 +664,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z1.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" @@ -649,23 +678,26 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z1.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[0]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z21.s, z9.s, z1.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1rqw z5.s, p6/z, [a_ptr1]\n" "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.s, z11.s, z1.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z20.s, z12.s, z1.s[1]\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" @@ -678,6 +710,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z1.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" @@ -693,14 +726,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z1.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z5.s[0]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z21.s, z9.s, z5.s[0]\n" @@ -708,13 +740,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z5.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" @@ -722,11 +754,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z14.s, z5.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" @@ -735,10 +768,28 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z22.s, z10.s, z5.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1w z20.s, p0, [c_ptr1]\n" @@ -748,7 +799,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -760,110 +811,97 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "c_ptr2 .req X3\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "whilelt p6.s, %[temp], %[leftovers]\n" "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" "ptrue p7.s\n" "whilelt p1.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "mov z21.d, z17.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.d, z18.d\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z23.d, z19.d\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z24.d, z16.d\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.s, #0\n" + "mov z26.d, z18.d\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.s, #0\n" + "mov z27.d, z19.d\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z24.s, #0\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z25.s, #0\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z26.s, #0\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" - "b 2f\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "2:\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "subs %[loops], %[loops], #0x1\n" "fmla z22.s, z10.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z26.s, z10.s, z2.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "subs %[loops], %[loops], #0x1\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" @@ -887,8 +925,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z21.s, z9.s, z1.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -955,8 +993,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "fmla z21.s, z9.s, z5.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z21.s, z9.s, z5.s[2]\n" "fmla z25.s, z9.s, z6.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" @@ -982,19 +1020,19 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" @@ -1026,8 +1064,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z21.s, z9.s, z1.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -1059,10 +1097,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.s, z8.s, z6.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.s, z9.s, z5.s[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" @@ -1090,6 +1131,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z27.s, z15.s, z6.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z17.s, z9.s, z4.s[2]\n" @@ -1113,14 +1155,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" "fmla z24.s, z8.s, z2.s[0]\n" "fmla z17.s, z9.s, z0.s[0]\n" @@ -1132,13 +1173,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z17.s, z13.s, z0.s[1]\n" @@ -1150,11 +1191,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" @@ -1167,24 +1209,28 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p6/z, [a_ptr1]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p6/z, [a_ptr2]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[0]\n" + "ld1rqw z5.s, p6/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1rqw z6.s, p6/z, [a_ptr2]\n" "fmla z25.s, z9.s, z2.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.s, z10.s, z1.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1205,6 +1251,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z27.s, z15.s, z2.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z17.s, z9.s, z0.s[2]\n" @@ -1228,14 +1275,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z5.s[0]\n" "fmla z24.s, z8.s, z6.s[0]\n" "fmla z17.s, z9.s, z4.s[0]\n" @@ -1247,13 +1293,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z17.s, z13.s, z4.s[1]\n" @@ -1265,11 +1311,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" @@ -1282,14 +1329,40 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" "st1w z24.s, p0, [c_ptr2]\n" @@ -1301,7 +1374,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1316,132 +1389,115 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "c_ptr3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "whilelt p6.s, %[temp], %[leftovers]\n" "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" "ptrue p7.s\n" "whilelt p1.s, %[temp], %[width]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" "incw %[temp], all, mul #1\n" "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "cbnz %[append], 1f\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "mov z20.d, z16.d\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "mov z21.d, z17.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.d, z18.d\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z23.d, z19.d\n" "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "mov z20.s, #0\n" + "mov z24.d, z16.d\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "mov z21.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.s, #0\n" + "mov z26.d, z18.d\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.s, #0\n" + "mov z27.d, z19.d\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.s, #0\n" + "mov z28.d, z16.d\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "mov z25.s, #0\n" + "mov z29.d, z17.d\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "mov z26.s, #0\n" + "mov z30.d, z18.d\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "mov z27.s, #0\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "mov z28.s, #0\n" + "mov z31.d, z19.d\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "b 2f\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" + "cbz %[loops], 2f\n" + "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z28.s, p7/m, z28.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z29.s, p7/m, z29.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "2:\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 2f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z28.s, z8.s, z3.s[0]\n" - "ld1rqw z7.s, p7/z, [a_ptr3]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z7.s, p7/z, [a_ptr3]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "subs %[loops], %[loops], #0x1\n" "fmla z29.s, z9.s, z3.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z22.s, z10.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z26.s, z10.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "add a_ptr3, a_ptr3, #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" @@ -1471,8 +1527,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1556,8 +1612,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z28.s, z8.s, z7.s[2]\n" - "fmla z17.s, z9.s, z4.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" "fmla z25.s, z9.s, z6.s[2]\n" "fmla z29.s, z9.s, z7.s[2]\n" @@ -1591,21 +1647,21 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" + "b.ne 3b\n" + "2:\n" + "cbz %[regs], 4f\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p7/z, [a_ptr1]\n" + "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p7/z, [a_ptr2]\n" + "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z28.s, z8.s, z3.s[0]\n" - "ld1rqw z7.s, p7/z, [a_ptr3]\n" + "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z17.s, z9.s, z0.s[0]\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "ld1rqw z7.s, p7/z, [a_ptr3]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "fmla z29.s, z9.s, z3.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1644,8 +1700,8 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1686,11 +1742,15 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z24.s, z8.s, z6.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z28.s, z8.s, z7.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z21.s, z9.s, z5.s[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" "fmla z29.s, z9.s, z7.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1725,6 +1785,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z31.s, z15.s, z7.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z28.s, z8.s, z7.s[2]\n" @@ -1756,14 +1817,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" "fmla z24.s, z8.s, z2.s[0]\n" "fmla z28.s, z8.s, z3.s[0]\n" @@ -1779,13 +1839,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" @@ -1801,11 +1861,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" "fmla z31.s, z15.s, z3.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" @@ -1822,25 +1883,30 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" "fmla z31.s, z11.s, z3.s[2]\n" - "b 6f\n" - "5:\n" + "b 5f\n" + "4:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" - "ld1rqw z5.s, p6/z, [a_ptr1]\n" + "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" "fmla z24.s, z8.s, z2.s[0]\n" - "ld1rqw z6.s, p6/z, [a_ptr2]\n" + "ld1rqw z5.s, p6/z, [a_ptr1]\n" "fmla z28.s, z8.s, z3.s[0]\n" - "ld1rqw z7.s, p6/z, [a_ptr3]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1w z8.s, p7/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[0]\n" + "ld1rqw z6.s, p6/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" + "ld1rqw z7.s, p6/z, [a_ptr3]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.s, z9.s, z3.s[0]\n" "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.s, z10.s, z1.s[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -1869,6 +1935,7 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z31.s, z15.s, z3.s[1]\n" "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" @@ -1900,14 +1967,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" "fmla z31.s, z15.s, z3.s[3]\n" - "cbz %[blocks], 6f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "cbz %[blocks], 5f\n" + "ld1w z8.s, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z5.s[0]\n" "fmla z24.s, z8.s, z6.s[0]\n" "fmla z28.s, z8.s, z7.s[0]\n" @@ -1923,13 +1989,13 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" "fmla z31.s, z11.s, z7.s[0]\n" - "b.eq 6f\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "b.eq 5f\n" + "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z28.s, z12.s, z7.s[1]\n" @@ -1945,11 +2011,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" "fmla z31.s, z15.s, z7.s[1]\n" - "b.eq 6f\n" - "ld1w z8.s, p7/z, [%[b_ptr0]]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" + "b.eq 5f\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" @@ -1966,17 +2033,51 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" "fmla z31.s, z11.s, z7.s[2]\n" - "6:\n" + "5:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmax z28.s, p7/m, z28.s, z14.s\n" + "fmax z29.s, p7/m, z29.s, z14.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" + "fmax z30.s, p7/m, z30.s, z14.s\n" + "fmin z28.s, p7/m, z28.s, z15.s\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.s, p7/m, z29.s, z15.s\n" + "fmax z31.s, p7/m, z31.s, z14.s\n" + "fmin z30.s, p7/m, z30.s, z15.s\n" "st1w z24.s, p0, [c_ptr2]\n" + "fmin z31.s, p7/m, z31.s, z15.s\n" "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n" "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n" "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n" @@ -1991,11 +2092,12 @@ void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp index ffd7918b7a..d8422105cc 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class hybrid_s8s32_dot_4VLx4 { @@ -40,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length<int32_t>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_s8s32_dot_4VLx4; - hybrid_s8s32_dot_4VLx4(const CPUInfo *ci) - { - - } + hybrid_s8s32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp index 673f186524..d57557a242 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,14 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -46,17 +46,16 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 const unsigned long ldab = lda * sizeof(int8_t); int32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(int32_t); for (int x0=0; x0<N; x0+=(4 * get_vector_length<int32_t>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<int32_t>())); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = blocks_count; const int8_t *a_ptr0 = a_ptr0_base; const int8_t *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(int32_t); switch(M-y) { case 1: @@ -70,7 +69,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -84,30 +83,23 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -224,13 +216,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" - "sdot z17.s, z13.b, z4.b[1]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "sdot z18.s, z14.b, z4.b[1]\n" + "sdot z17.s, z13.b, z4.b[1]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "sdot z19.s, z15.b, z4.b[1]\n" + "sdot z18.s, z14.b, z4.b[1]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "sdot z19.s, z15.b, z4.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -241,45 +232,48 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" + "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" "sdot z17.s, z13.b, z4.b[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z18.s, z14.b, z4.b[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z19.s, z15.b, z4.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z0.b[0]\n" "sdot z18.s, z10.b, z0.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z17.s, z13.b, z0.b[1]\n" "sdot z18.s, z14.b, z0.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z0.b[2]\n" "sdot z18.s, z10.b, z0.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" @@ -289,65 +283,67 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "sdot z16.s, z12.b, z0.b[1]\n" + "sdot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "sdot z17.s, z13.b, z0.b[1]\n" + "sdot z16.s, z12.b, z0.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "sdot z18.s, z14.b, z0.b[1]\n" + "sdot z17.s, z13.b, z0.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "sdot z19.s, z15.b, z0.b[1]\n" + "sdot z18.s, z14.b, z0.b[1]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "sdot z16.s, z8.b, z0.b[2]\n" + "sdot z19.s, z15.b, z0.b[1]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "sdot z17.s, z9.b, z0.b[2]\n" + "sdot z16.s, z8.b, z0.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "sdot z17.s, z9.b, z0.b[2]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "sdot z18.s, z10.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z19.s, z11.b, z0.b[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z18.s, z10.b, z4.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z17.s, z13.b, z4.b[1]\n" "sdot z18.s, z14.b, z4.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z17.s, z9.b, z4.b[2]\n" "sdot z18.s, z10.b, z4.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z17.s, z13.b, z4.b[3]\n" "sdot z18.s, z14.b, z4.b[3]\n" @@ -359,7 +355,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -378,7 +374,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -397,41 +393,30 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -601,9 +586,11 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z20.s, z8.b, z5.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z4.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z4.b[0]\n" @@ -622,6 +609,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z5.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" "sdot z21.s, z9.b, z5.b[2]\n" @@ -638,13 +626,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" "sdot z17.s, z9.b, z0.b[0]\n" "sdot z21.s, z9.b, z1.b[0]\n" @@ -653,12 +640,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z1.b[1]\n" "sdot z17.s, z13.b, z0.b[1]\n" "sdot z21.s, z13.b, z1.b[1]\n" @@ -667,12 +654,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" "sdot z21.s, z9.b, z1.b[2]\n" @@ -681,10 +669,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z20.s, z12.b, z1.b[3]\n" "sdot z17.s, z13.b, z0.b[3]\n" @@ -698,19 +686,21 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "sdot z18.s, z10.b, z0.b[0]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "sdot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z23.s, z11.b, z1.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "sdot z20.s, z12.b, z1.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "sdot z17.s, z13.b, z0.b[1]\n" @@ -723,6 +713,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z1.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" "sdot z21.s, z9.b, z1.b[2]\n" @@ -739,13 +730,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[0]\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" @@ -754,12 +744,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z5.b[1]\n" "sdot z17.s, z13.b, z4.b[1]\n" "sdot z21.s, z13.b, z5.b[1]\n" @@ -768,12 +758,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" "sdot z21.s, z9.b, z5.b[2]\n" @@ -782,10 +773,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z20.s, z12.b, z5.b[3]\n" "sdot z17.s, z13.b, z4.b[3]\n" @@ -807,7 +798,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -830,7 +821,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -852,54 +843,39 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "mov z25.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z26.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1120,10 +1096,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z4.b[0]\n" "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "sdot z25.s, z9.b, z6.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z4.b[0]\n" @@ -1151,6 +1130,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z6.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" @@ -1175,13 +1155,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" "sdot z24.s, z8.b, z2.b[0]\n" "sdot z17.s, z9.b, z0.b[0]\n" @@ -1194,12 +1173,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z1.b[1]\n" "sdot z24.s, z12.b, z2.b[1]\n" "sdot z17.s, z13.b, z0.b[1]\n" @@ -1212,12 +1191,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" @@ -1230,10 +1210,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z20.s, z12.b, z1.b[3]\n" "sdot z24.s, z12.b, z2.b[3]\n" @@ -1253,18 +1233,21 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "sdot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "sdot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z0.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1285,6 +1268,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z2.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z17.s, z9.b, z0.b[2]\n" @@ -1309,13 +1293,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[0]\n" "sdot z24.s, z8.b, z6.b[0]\n" "sdot z17.s, z9.b, z4.b[0]\n" @@ -1328,12 +1311,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z5.b[1]\n" "sdot z24.s, z12.b, z6.b[1]\n" "sdot z17.s, z13.b, z4.b[1]\n" @@ -1346,12 +1329,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z17.s, z9.b, z4.b[2]\n" @@ -1364,10 +1348,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z20.s, z12.b, z5.b[3]\n" "sdot z24.s, z12.b, z6.b[3]\n" @@ -1399,7 +1383,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1427,7 +1411,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -1451,68 +1435,49 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "mov z26.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z27.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" + "mov z28.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" + "mov z29.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" + "mov z30.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" + "mov z31.s, #0\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1786,11 +1751,15 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z16.s, z8.b, z4.b[0]\n" "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z28.s, z8.b, z7.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "sdot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "sdot z25.s, z9.b, z6.b[0]\n" "sdot z29.s, z9.b, z7.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1825,6 +1794,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z31.s, z15.b, z7.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z28.s, z8.b, z7.b[2]\n" @@ -1857,13 +1827,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[0]\n" "sdot z24.s, z8.b, z2.b[0]\n" "sdot z28.s, z8.b, z3.b[0]\n" @@ -1880,12 +1849,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z1.b[1]\n" "sdot z24.s, z12.b, z2.b[1]\n" "sdot z28.s, z12.b, z3.b[1]\n" @@ -1902,12 +1871,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z2.b[1]\n" "sdot z31.s, z15.b, z3.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z28.s, z8.b, z3.b[2]\n" @@ -1924,10 +1894,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z2.b[2]\n" "sdot z31.s, z11.b, z3.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z0.b[3]\n" "sdot z20.s, z12.b, z1.b[3]\n" "sdot z24.s, z12.b, z2.b[3]\n" @@ -1953,17 +1923,21 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" "sdot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z7.b, p6/z, [a_ptr3]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z7.b, p6/z, [a_ptr3]\n" "sdot z25.s, z9.b, z2.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "sdot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "sdot z18.s, z10.b, z0.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "sdot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z19.s, z11.b, z0.b[0]\n" @@ -1992,6 +1966,7 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z31.s, z15.b, z3.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "sdot z20.s, z8.b, z1.b[2]\n" "sdot z24.s, z8.b, z2.b[2]\n" "sdot z28.s, z8.b, z3.b[2]\n" @@ -2024,13 +1999,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z2.b[3]\n" "sdot z31.s, z15.b, z3.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "sdot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[0]\n" "sdot z24.s, z8.b, z6.b[0]\n" "sdot z28.s, z8.b, z7.b[0]\n" @@ -2047,12 +2021,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z6.b[0]\n" "sdot z31.s, z11.b, z7.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "sdot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "sdot z20.s, z12.b, z5.b[1]\n" "sdot z24.s, z12.b, z6.b[1]\n" "sdot z28.s, z12.b, z7.b[1]\n" @@ -2069,12 +2043,13 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z15.b, z6.b[1]\n" "sdot z31.s, z15.b, z7.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "sdot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "sdot z20.s, z8.b, z5.b[2]\n" "sdot z24.s, z8.b, z6.b[2]\n" "sdot z28.s, z8.b, z7.b[2]\n" @@ -2091,10 +2066,10 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 "sdot z27.s, z11.b, z6.b[2]\n" "sdot z31.s, z11.b, z7.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "sdot z16.s, z12.b, z4.b[3]\n" "sdot z20.s, z12.b, z5.b[3]\n" "sdot z24.s, z12.b, z6.b[3]\n" @@ -2136,11 +2111,12 @@ void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32 ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp index 2e51349d35..5dab1da135 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class hybrid_u8u32_dot_4VLx4 { @@ -40,7 +40,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -53,20 +53,32 @@ public: return get_vector_length<uint32_t>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return true; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {}; // Default to the generic kernel kern_type kernel=sve_hybrid_u8u32_dot_4VLx4; - hybrid_u8u32_dot_4VLx4(const CPUInfo *ci) - { - - } + hybrid_u8u32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp index d34d0e5fc7..bf3e8ca26a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,14 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -46,17 +46,16 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin const unsigned long ldab = lda * sizeof(uint8_t); uint32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(uint32_t); for (int x0=0; x0<N; x0+=(4 * get_vector_length<uint32_t>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<uint32_t>())); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = blocks_count; const uint8_t *a_ptr0 = a_ptr0_base; const uint8_t *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(uint32_t); switch(M-y) { case 1: @@ -70,7 +69,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -84,30 +83,23 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -224,13 +216,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" - "udot z17.s, z13.b, z4.b[1]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z18.s, z14.b, z4.b[1]\n" + "udot z17.s, z13.b, z4.b[1]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "udot z19.s, z15.b, z4.b[1]\n" + "udot z18.s, z14.b, z4.b[1]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "udot z19.s, z15.b, z4.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -241,45 +232,48 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" + "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" "udot z17.s, z13.b, z4.b[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z18.s, z14.b, z4.b[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z19.s, z15.b, z4.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" @@ -289,65 +283,67 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "udot z16.s, z12.b, z0.b[1]\n" + "udot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "udot z17.s, z13.b, z0.b[1]\n" + "udot z16.s, z12.b, z0.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "udot z18.s, z14.b, z0.b[1]\n" + "udot z17.s, z13.b, z0.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "udot z19.s, z15.b, z0.b[1]\n" + "udot z18.s, z14.b, z0.b[1]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "udot z16.s, z8.b, z0.b[2]\n" + "udot z19.s, z15.b, z0.b[1]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "udot z17.s, z9.b, z0.b[2]\n" + "udot z16.s, z8.b, z0.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "udot z17.s, z9.b, z0.b[2]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z18.s, z10.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z19.s, z11.b, z0.b[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" @@ -359,7 +355,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -378,7 +374,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -397,41 +393,30 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -601,9 +586,11 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z20.s, z8.b, z5.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z4.b[0]\n" @@ -622,6 +609,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z21.s, z9.b, z5.b[2]\n" @@ -638,13 +626,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z17.s, z9.b, z0.b[0]\n" "udot z21.s, z9.b, z1.b[0]\n" @@ -653,12 +640,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z17.s, z13.b, z0.b[1]\n" "udot z21.s, z13.b, z1.b[1]\n" @@ -667,12 +654,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z21.s, z9.b, z1.b[2]\n" @@ -681,10 +669,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" @@ -698,19 +686,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "udot z18.s, z10.b, z0.b[0]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z23.s, z11.b, z1.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z20.s, z12.b, z1.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "udot z17.s, z13.b, z0.b[1]\n" @@ -723,6 +713,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z21.s, z9.b, z1.b[2]\n" @@ -739,13 +730,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -754,12 +744,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z17.s, z13.b, z4.b[1]\n" "udot z21.s, z13.b, z5.b[1]\n" @@ -768,12 +758,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z21.s, z9.b, z5.b[2]\n" @@ -782,10 +773,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z17.s, z13.b, z4.b[3]\n" @@ -807,7 +798,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -830,7 +821,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -852,54 +843,39 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "mov z25.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z26.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1120,10 +1096,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z4.b[0]\n" "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "udot z25.s, z9.b, z6.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z4.b[0]\n" @@ -1151,6 +1130,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" @@ -1175,13 +1155,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z24.s, z8.b, z2.b[0]\n" "udot z17.s, z9.b, z0.b[0]\n" @@ -1194,12 +1173,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z24.s, z12.b, z2.b[1]\n" "udot z17.s, z13.b, z0.b[1]\n" @@ -1212,12 +1191,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" @@ -1230,10 +1210,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z24.s, z12.b, z2.b[3]\n" @@ -1253,18 +1233,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1285,6 +1268,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" @@ -1309,13 +1293,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z24.s, z8.b, z6.b[0]\n" "udot z17.s, z9.b, z4.b[0]\n" @@ -1328,12 +1311,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z24.s, z12.b, z6.b[1]\n" "udot z17.s, z13.b, z4.b[1]\n" @@ -1346,12 +1329,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" @@ -1364,10 +1348,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z24.s, z12.b, z6.b[3]\n" @@ -1399,7 +1383,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1427,7 +1411,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -1451,68 +1435,49 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "mov z26.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z27.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" + "mov z28.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" + "mov z29.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" + "mov z30.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" + "mov z31.s, #0\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1786,11 +1751,15 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z4.b[0]\n" "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z28.s, z8.b, z7.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "udot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "udot z25.s, z9.b, z6.b[0]\n" "udot z29.s, z9.b, z7.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1825,6 +1794,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z31.s, z15.b, z7.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z28.s, z8.b, z7.b[2]\n" @@ -1857,13 +1827,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z24.s, z8.b, z2.b[0]\n" "udot z28.s, z8.b, z3.b[0]\n" @@ -1880,12 +1849,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z24.s, z12.b, z2.b[1]\n" "udot z28.s, z12.b, z3.b[1]\n" @@ -1902,12 +1871,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z28.s, z8.b, z3.b[2]\n" @@ -1924,10 +1894,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z24.s, z12.b, z2.b[3]\n" @@ -1953,17 +1923,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z7.b, p6/z, [a_ptr3]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z7.b, p6/z, [a_ptr3]\n" "udot z25.s, z9.b, z2.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z0.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z26.s, z10.b, z2.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -1992,6 +1966,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z31.s, z15.b, z3.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z28.s, z8.b, z3.b[2]\n" @@ -2024,13 +1999,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z24.s, z8.b, z6.b[0]\n" "udot z28.s, z8.b, z7.b[0]\n" @@ -2047,12 +2021,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z24.s, z12.b, z6.b[1]\n" "udot z28.s, z12.b, z7.b[1]\n" @@ -2069,12 +2043,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z28.s, z8.b, z7.b[2]\n" @@ -2091,10 +2066,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z24.s, z12.b, z6.b[3]\n" @@ -2136,11 +2111,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp index 2ca4ce25e8..a74f9efb87 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_fp16_mla_3VLx8; - interleaved_fp16_mla_3VLx8(const CPUInfo *ci) - { - - } + interleaved_fp16_mla_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp index 8c1fe6d0b6..1e73d8e185 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_fp32_mla_3VLx8; - interleaved_fp32_mla_3VLx8(const CPUInfo *ci) - { - - } + interleaved_fp32_mla_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp index cbb21387b1..8668ea7fbc 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_s8s32_dot_3VLx8; - interleaved_s8s32_dot_3VLx8(const CPUInfo *ci) - { - - } + interleaved_s8s32_dot_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp index 99c039e121..d3c8851154 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 Arm Limited. + * Copyright (c) 2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -61,10 +61,7 @@ public: kern_type kernel=sve_interleaved_u8u32_dot_3VLx8; - interleaved_u8u32_dot_3VLx8(const CPUInfo *ci) - { - - } + interleaved_u8u32_dot_3VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp index 6cce601dcc..741f200d25 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,11 @@ #ifdef __ARM_FEATURE_SVE - - - namespace arm_gemm { // Actual kernel implementations -void sve_native_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, __fp16, int, int, int); +void sve_native_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); class native_fp16_mla_4VLx4 { @@ -40,10 +37,10 @@ public: typedef __fp16 operand_type; typedef __fp16 result_type; - typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, __fp16, int, int, int); + typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, int ldb, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +50,32 @@ public: return get_vector_length<__fp16>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + // Default to the generic kernel kern_type kernel=sve_native_fp16_mla_4VLx4; - native_fp16_mla_4VLx4(const CPUInfo *ci) - { - - } + native_fp16_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp index f1aaeb13ee..14dd38bd25 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,30 +25,49 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ldb, __fp16 *C, int ldc, __fp16 beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ldb, __fp16 *C, int ldc, int M, int N, int K, const __fp16 *bias, Activation act, bool append) { const long loops_count = ((K + 8) / 16) - 1; K -= loops_count * 16; const long regs_count = (K / 8) - 1; K -= (regs_count + 1) * 8; const long leftovers = K; + __fp16 nullbias[512]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<__fp16>() * sizeof(__fp16))); + } + __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity()); + __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity()); + const __fp16 * const minptr = &minval; + const __fp16 * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<__fp16>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const __fp16 * const a_ptr0_base = A + (y * lda); const unsigned long ldab = lda * sizeof(__fp16); __fp16 *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(__fp16); for (int x0=0; x0<N; x0+=(4 * get_vector_length<__fp16>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<__fp16>())); - const __fp16 *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -56,6 +75,8 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld const __fp16 *a_ptr0 = a_ptr0_base; const __fp16 *b_ptr0 = B + x0; long ldbb = ldb * sizeof(__fp16); + const unsigned long ldcb = ldc * sizeof(__fp16); + const __fp16 *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -64,52 +85,34 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "inch %[temp], all, mul #1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z18.h, #0\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.h, #0\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "whilelt p2.h, %[temp], %[width]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" + "inch %[temp], all, mul #1\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "cbz %[loops], 2f\n" - "3:\n" + "whilelt p3.h, %[temp], %[width]\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" + "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z9.h, z0.h[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" "fmla z18.h, z10.h, z0.h[0]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "subs %[loops], %[loops], #0x1\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -119,70 +122,69 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" - "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z19.h, z11.h, z0.h[2]\n" + "fmla z17.h, z9.h, z0.h[2]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[2]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[2]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z12.h, z0.h[3]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z13.h, z0.h[3]\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z14.h, z0.h[3]\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z15.h, z0.h[3]\n" + "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z8.h, z0.h[4]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z9.h, z0.h[4]\n" + "fmla z16.h, z8.h, z0.h[4]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[4]\n" + "fmla z17.h, z9.h, z0.h[4]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[4]\n" + "fmla z18.h, z10.h, z0.h[4]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[4]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z12.h, z0.h[5]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z13.h, z0.h[5]\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z14.h, z0.h[5]\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z15.h, z0.h[5]\n" + "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z8.h, z0.h[6]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z9.h, z0.h[6]\n" + "fmla z16.h, z8.h, z0.h[6]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[6]\n" + "fmla z17.h, z9.h, z0.h[6]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[6]\n" + "fmla z18.h, z10.h, z0.h[6]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[6]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z12.h, z0.h[7]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z13.h, z0.h[7]\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z14.h, z0.h[7]\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z15.h, z0.h[7]\n" + "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z4.h[0]\n" "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z17.h, z9.h, z4.h[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z4.h[0]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z19.h, z11.h, z4.h[0]\n" + "fmla z18.h, z10.h, z4.h[0]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.h, z11.h, z4.h[0]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" @@ -247,94 +249,95 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z18.h, z14.h, z4.h[7]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z15.h, z4.h[7]\n" + "b.ne 2b\n" + "1:\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "b.ne 3b\n" - "2:\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z16.h, z12.h, z0.h[1]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z17.h, z13.h, z0.h[1]\n" + "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[1]\n" + "fmla z17.h, z13.h, z0.h[1]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[1]\n" + "fmla z18.h, z14.h, z0.h[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.h, z8.h, z0.h[2]\n" + "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z18.h, z10.h, z0.h[2]\n" + "fmla z17.h, z9.h, z0.h[2]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[2]\n" + "fmla z18.h, z10.h, z0.h[2]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[3]\n" + "fmla z19.h, z11.h, z0.h[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[3]\n" + "fmla z16.h, z12.h, z0.h[3]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[3]\n" + "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[3]\n" + "fmla z18.h, z14.h, z0.h[3]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[3]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[4]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[4]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[4]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[4]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[5]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[5]\n" + "fmla z16.h, z12.h, z0.h[5]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[5]\n" + "fmla z17.h, z13.h, z0.h[5]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[5]\n" + "fmla z18.h, z14.h, z0.h[5]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[5]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[6]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[6]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[6]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[6]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[7]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[7]\n" + "fmla z16.h, z12.h, z0.h[7]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[7]\n" + "fmla z17.h, z13.h, z0.h[7]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[7]\n" + "fmla z18.h, z14.h, z0.h[7]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[7]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z4.h[0]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.h, z9.h, z4.h[0]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.h, z10.h, z4.h[0]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z19.h, z11.h, z4.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z16.h, z12.h, z4.h[1]\n" - "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n" + "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.h, z13.h, z4.h[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.h, z14.h, z4.h[1]\n" @@ -392,7 +395,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[7]\n" "fmla z18.h, z14.h, z4.h[7]\n" "fmla z19.h, z15.h, z4.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -403,7 +406,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[0]\n" "fmla z18.h, z10.h, z0.h[0]\n" "fmla z19.h, z11.h, z0.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -414,7 +417,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z0.h[1]\n" "fmla z18.h, z14.h, z0.h[1]\n" "fmla z19.h, z15.h, z0.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -425,7 +428,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[2]\n" "fmla z18.h, z10.h, z0.h[2]\n" "fmla z19.h, z11.h, z0.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -436,7 +439,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z0.h[3]\n" "fmla z18.h, z14.h, z0.h[3]\n" "fmla z19.h, z15.h, z0.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -447,7 +450,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[4]\n" "fmla z18.h, z10.h, z0.h[4]\n" "fmla z19.h, z11.h, z0.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -458,7 +461,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z0.h[5]\n" "fmla z18.h, z14.h, z0.h[5]\n" "fmla z19.h, z15.h, z0.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -468,73 +471,73 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z0.h[6]\n" "fmla z18.h, z10.h, z0.h[6]\n" "fmla z19.h, z11.h, z0.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.h, z9.h, z0.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z16.h, z12.h, z0.h[1]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z17.h, z13.h, z0.h[1]\n" + "fmla z16.h, z12.h, z0.h[1]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z18.h, z14.h, z0.h[1]\n" + "fmla z17.h, z13.h, z0.h[1]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[1]\n" + "fmla z18.h, z14.h, z0.h[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.h, z8.h, z0.h[2]\n" + "fmla z19.h, z15.h, z0.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[2]\n" + "fmla z16.h, z8.h, z0.h[2]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z18.h, z10.h, z0.h[2]\n" + "fmla z17.h, z9.h, z0.h[2]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[2]\n" + "fmla z18.h, z10.h, z0.h[2]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[3]\n" + "fmla z19.h, z11.h, z0.h[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[3]\n" + "fmla z16.h, z12.h, z0.h[3]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[3]\n" + "fmla z17.h, z13.h, z0.h[3]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[3]\n" + "fmla z18.h, z14.h, z0.h[3]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[3]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[4]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[4]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[4]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[4]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z16.h, z12.h, z0.h[5]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z17.h, z13.h, z0.h[5]\n" + "fmla z16.h, z12.h, z0.h[5]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z14.h, z0.h[5]\n" + "fmla z17.h, z13.h, z0.h[5]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z15.h, z0.h[5]\n" + "fmla z18.h, z14.h, z0.h[5]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z15.h, z0.h[5]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.h, z8.h, z0.h[6]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.h, z9.h, z0.h[6]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z18.h, z10.h, z0.h[6]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z19.h, z11.h, z0.h[6]\n" + "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.h, z12.h, z0.h[7]\n" "fmla z17.h, z13.h, z0.h[7]\n" "fmla z18.h, z14.h, z0.h[7]\n" "fmla z19.h, z15.h, z0.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -545,7 +548,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[0]\n" "fmla z18.h, z10.h, z4.h[0]\n" "fmla z19.h, z11.h, z4.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -556,7 +559,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[1]\n" "fmla z18.h, z14.h, z4.h[1]\n" "fmla z19.h, z15.h, z4.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -567,7 +570,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[2]\n" "fmla z18.h, z10.h, z4.h[2]\n" "fmla z19.h, z11.h, z4.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -578,7 +581,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[3]\n" "fmla z18.h, z14.h, z4.h[3]\n" "fmla z19.h, z15.h, z4.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -589,7 +592,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[4]\n" "fmla z18.h, z10.h, z4.h[4]\n" "fmla z19.h, z11.h, z4.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -600,7 +603,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z13.h, z4.h[5]\n" "fmla z18.h, z14.h, z4.h[5]\n" "fmla z19.h, z15.h, z4.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -610,14 +613,24 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z17.h, z9.h, z4.h[6]\n" "fmla z18.h, z10.h, z4.h[6]\n" "fmla z19.h, z11.h, z4.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -631,66 +644,33 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "inch %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z19.h, #0\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.h, #0\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.h, #0\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.h, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z23.h, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" - "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" - "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" - "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" + "whilelt p2.h, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "inch %[temp], all, mul #1\n" + "add a_ptr1, a_ptr1, #0x10\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "whilelt p3.h, %[temp], %[width]\n" + "mov z22.d, z18.d\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "mov z23.d, z19.d\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "3:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" @@ -906,23 +886,23 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" + "b.ne 2b\n" + "1:\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p7/z, [a_ptr1]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z22.h, z10.h, z1.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z22.h, z10.h, z1.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" @@ -1026,9 +1006,11 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z21.h, z9.h, z5.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z22.h, z10.h, z5.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z23.h, z11.h, z5.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z4.h[1]\n" @@ -1112,7 +1094,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[7]\n" "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1127,7 +1109,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[0]\n" "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1142,7 +1124,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[1]\n" "fmla z19.h, z15.h, z0.h[1]\n" "fmla z23.h, z15.h, z1.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1157,7 +1139,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[2]\n" "fmla z19.h, z11.h, z0.h[2]\n" "fmla z23.h, z11.h, z1.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1172,7 +1154,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[3]\n" "fmla z19.h, z15.h, z0.h[3]\n" "fmla z23.h, z15.h, z1.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1187,7 +1169,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[4]\n" "fmla z19.h, z11.h, z0.h[4]\n" "fmla z23.h, z11.h, z1.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1202,7 +1184,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[5]\n" "fmla z19.h, z15.h, z0.h[5]\n" "fmla z23.h, z15.h, z1.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1216,22 +1198,22 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z1.h[6]\n" "fmla z19.h, z11.h, z0.h[6]\n" "fmla z23.h, z11.h, z1.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z22.h, z10.h, z1.h[0]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.h, z11.h, z0.h[0]\n" + "fmla z22.h, z10.h, z1.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.h, z11.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" @@ -1239,6 +1221,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z20.h, z12.h, z1.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z13.h, z0.h[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z21.h, z13.h, z1.h[1]\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z14.h, z0.h[1]\n" @@ -1315,7 +1298,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z1.h[7]\n" "fmla z19.h, z15.h, z0.h[7]\n" "fmla z23.h, z15.h, z1.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1330,7 +1313,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[0]\n" "fmla z19.h, z11.h, z4.h[0]\n" "fmla z23.h, z11.h, z5.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1345,7 +1328,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[1]\n" "fmla z19.h, z15.h, z4.h[1]\n" "fmla z23.h, z15.h, z5.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1360,7 +1343,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[2]\n" "fmla z19.h, z11.h, z4.h[2]\n" "fmla z23.h, z11.h, z5.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1375,7 +1358,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[3]\n" "fmla z19.h, z15.h, z4.h[3]\n" "fmla z23.h, z15.h, z5.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -1390,7 +1373,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[4]\n" "fmla z19.h, z11.h, z4.h[4]\n" "fmla z23.h, z11.h, z5.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -1405,7 +1388,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z14.h, z5.h[5]\n" "fmla z19.h, z15.h, z4.h[5]\n" "fmla z23.h, z15.h, z5.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1419,10 +1402,28 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z22.h, z10.h, z5.h[6]\n" "fmla z19.h, z11.h, z4.h[6]\n" "fmla z23.h, z11.h, z5.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1h z20.h, p0, [c_ptr1]\n" @@ -1432,7 +1433,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -1450,82 +1451,39 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" - "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" - "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z20.h, #0\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.h, #0\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.h, #0\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.h, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z24.h, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "mov z25.h, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z26.h, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z27.h, #0\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" - "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" - "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" - "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" - "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" - "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" - "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" + "inch %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" + "whilelt p2.h, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "mov z25.d, z17.d\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "inch %[temp], all, mul #1\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "whilelt p3.h, %[temp], %[width]\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" + "mov z26.d, z18.d\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "mov z23.d, z19.d\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "3:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" @@ -1551,9 +1509,9 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z27.h, z11.h, z2.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.h, z12.h, z0.h[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z20.h, z12.h, z1.h[1]\n" "add a_ptr2, a_ptr2, #0x20\n" + "fmla z20.h, z12.h, z1.h[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.h, z12.h, z2.h[1]\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z13.h, z0.h[1]\n" @@ -1808,23 +1766,23 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" + "b.ne 2b\n" + "1:\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p7/z, [a_ptr1]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p7/z, [a_ptr2]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z25.h, z9.h, z2.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[0]\n" "fmla z22.h, z10.h, z1.h[0]\n" "fmla z26.h, z10.h, z2.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -1960,10 +1918,13 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z24.h, z8.h, z6.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z22.h, z10.h, z5.h[0]\n" "fmla z26.h, z10.h, z6.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -2080,7 +2041,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[7]\n" "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2099,7 +2060,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[0]\n" "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2118,7 +2079,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[1]\n" "fmla z23.h, z15.h, z1.h[1]\n" "fmla z27.h, z15.h, z2.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2137,7 +2098,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[2]\n" "fmla z23.h, z11.h, z1.h[2]\n" "fmla z27.h, z11.h, z2.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2156,7 +2117,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[3]\n" "fmla z23.h, z15.h, z1.h[3]\n" "fmla z27.h, z15.h, z2.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2175,7 +2136,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[4]\n" "fmla z23.h, z11.h, z1.h[4]\n" "fmla z27.h, z11.h, z2.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2194,7 +2155,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[5]\n" "fmla z23.h, z15.h, z1.h[5]\n" "fmla z27.h, z15.h, z2.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -2212,26 +2173,28 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z0.h[6]\n" "fmla z23.h, z11.h, z1.h[6]\n" "fmla z27.h, z11.h, z2.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p6/z, [a_ptr2]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "fmla z18.h, z10.h, z0.h[0]\n" + "fmla z25.h, z9.h, z2.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.h, z10.h, z0.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" @@ -2344,7 +2307,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z0.h[7]\n" "fmla z23.h, z15.h, z1.h[7]\n" "fmla z27.h, z15.h, z2.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2363,7 +2326,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[0]\n" "fmla z23.h, z11.h, z5.h[0]\n" "fmla z27.h, z11.h, z6.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2382,7 +2345,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[1]\n" "fmla z23.h, z15.h, z5.h[1]\n" "fmla z27.h, z15.h, z6.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2401,7 +2364,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[2]\n" "fmla z23.h, z11.h, z5.h[2]\n" "fmla z27.h, z11.h, z6.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2420,7 +2383,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[3]\n" "fmla z23.h, z15.h, z5.h[3]\n" "fmla z27.h, z15.h, z6.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -2439,7 +2402,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[4]\n" "fmla z23.h, z11.h, z5.h[4]\n" "fmla z27.h, z11.h, z6.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -2458,7 +2421,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z15.h, z4.h[5]\n" "fmla z23.h, z15.h, z5.h[5]\n" "fmla z27.h, z15.h, z6.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -2476,14 +2439,40 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z19.h, z11.h, z4.h[6]\n" "fmla z23.h, z11.h, z5.h[6]\n" "fmla z27.h, z11.h, z6.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" "st1h z24.h, p0, [c_ptr2]\n" @@ -2495,7 +2484,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -2518,98 +2507,45 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "whilelt p0.h, %[temp], %[width]\n" "inch %[temp], all, mul #1\n" "ptrue p7.h\n" + "ld1h z16.h, p0/z, [%[biasptr]]\n" "whilelt p1.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p2.h, %[temp], %[width]\n" - "inch %[temp], all, mul #1\n" - "whilelt p3.h, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.h, #0\n" "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "mov z17.h, #0\n" + "inch %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "mov z18.h, #0\n" + "mov z28.d, z16.d\n" "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "mov z19.h, #0\n" "ld1rqh z3.h, p7/z, [a_ptr3]\n" - "mov z20.h, #0\n" + "whilelt p2.h, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "mov z21.h, #0\n" + "mov z25.d, z17.d\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.h, #0\n" + "mov z29.d, z17.d\n" + "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.h, #0\n" - "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.h, #0\n" + "inch %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z25.h, #0\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z26.h, #0\n" + "mov z26.d, z18.d\n" + "whilelt p3.h, %[temp], %[width]\n" + "mov z30.d, z18.d\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z27.h, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "mov z28.h, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z29.h, #0\n" - "ld1h z12.h, p0/z, [%[b_ptr0]]\n" - "mov z30.h, #0\n" - "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z31.h, #0\n" - "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rh z15.h, p7/z, [%[betaptr]]\n" - "ld1h z16.h, p0/z, [%[c_ptr0]]\n" - "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "fmul z16.h, p7/m, z16.h, z15.h\n" - "ld1h z20.h, p0/z, [c_ptr1]\n" - "fmul z17.h, p7/m, z17.h, z15.h\n" - "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z18.h, p7/m, z18.h, z15.h\n" - "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z19.h, p7/m, z19.h, z15.h\n" - "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z20.h, p7/m, z20.h, z15.h\n" - "ld1h z24.h, p0/z, [c_ptr2]\n" - "fmul z21.h, p7/m, z21.h, z15.h\n" - "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z22.h, p7/m, z22.h, z15.h\n" - "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z23.h, p7/m, z23.h, z15.h\n" - "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z24.h, p7/m, z24.h, z15.h\n" - "ld1h z28.h, p0/z, [c_ptr3]\n" - "fmul z25.h, p7/m, z25.h, z15.h\n" - "ld1h z29.h, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z26.h, p7/m, z26.h, z15.h\n" - "ld1h z30.h, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z27.h, p7/m, z27.h, z15.h\n" - "ld1h z31.h, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z28.h, p7/m, z28.h, z15.h\n" - "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n" - "fmul z29.h, p7/m, z29.h, z15.h\n" - "ld1rqh z1.h, p7/z, [a_ptr1]\n" - "fmul z30.h, p7/m, z30.h, z15.h\n" - "ld1rqh z2.h, p7/z, [a_ptr2]\n" - "fmul z31.h, p7/m, z31.h, z15.h\n" - "ld1rqh z3.h, p7/z, [a_ptr3]\n" - "ld1h z8.h, p0/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" - "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" + "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1h z11.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "mov z23.d, z19.d\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1h z13.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "mov z31.d, z19.d\n" "ld1h z14.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 2f\n" - "3:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.h, z8.h, z0.h[0]\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.h, z8.h, z1.h[0]\n" @@ -2959,23 +2895,23 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" "fmla z31.h, z15.h, z7.h[7]\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "fmla z16.h, z8.h, z0.h[0]\n" + "b.ne 2b\n" + "1:\n" "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.h, z8.h, z0.h[0]\n" "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p7/z, [a_ptr1]\n" - "fmla z28.h, z8.h, z3.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p7/z, [a_ptr2]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z28.h, z8.h, z3.h[0]\n" "ld1rqh z7.h, p7/z, [a_ptr3]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "fmla z25.h, z9.h, z2.h[0]\n" "fmla z29.h, z9.h, z3.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" @@ -3143,11 +3079,15 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z20.h, z8.h, z5.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.h, z8.h, z6.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z28.h, z8.h, z7.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "fmla z17.h, z9.h, z4.h[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.h, z9.h, z5.h[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.h, z9.h, z6.h[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z29.h, z9.h, z7.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z4.h[0]\n" @@ -3297,7 +3237,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[7]\n" "fmla z27.h, z15.h, z6.h[7]\n" "fmla z31.h, z15.h, z7.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3320,7 +3260,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[0]\n" "fmla z27.h, z11.h, z2.h[0]\n" "fmla z31.h, z11.h, z3.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3343,7 +3283,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[1]\n" "fmla z27.h, z15.h, z2.h[1]\n" "fmla z31.h, z15.h, z3.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3366,7 +3306,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[2]\n" "fmla z27.h, z11.h, z2.h[2]\n" "fmla z31.h, z11.h, z3.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3389,7 +3329,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[3]\n" "fmla z27.h, z15.h, z2.h[3]\n" "fmla z31.h, z15.h, z3.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3412,7 +3352,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[4]\n" "fmla z27.h, z11.h, z2.h[4]\n" "fmla z31.h, z11.h, z3.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3435,7 +3375,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[5]\n" "fmla z27.h, z15.h, z2.h[5]\n" "fmla z31.h, z15.h, z3.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -3457,27 +3397,30 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z1.h[6]\n" "fmla z27.h, z11.h, z2.h[6]\n" "fmla z31.h, z11.h, z3.h[6]\n" - "b 5f\n" - "4:\n" + "b 4f\n" + "3:\n" "fmla z16.h, z8.h, z0.h[0]\n" - "ld1h z15.h, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n" - "fmla z24.h, z8.h, z2.h[0]\n" + "fmla z20.h, z8.h, z1.h[0]\n" "ld1rqh z5.h, p6/z, [a_ptr1]\n" - "fmla z28.h, z8.h, z3.h[0]\n" + "fmla z24.h, z8.h, z2.h[0]\n" "ld1rqh z6.h, p6/z, [a_ptr2]\n" - "fmla z17.h, z9.h, z0.h[0]\n" + "fmla z28.h, z8.h, z3.h[0]\n" "ld1rqh z7.h, p6/z, [a_ptr3]\n" - "fmla z21.h, z9.h, z1.h[0]\n" + "fmla z17.h, z9.h, z0.h[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z25.h, z9.h, z2.h[0]\n" + "fmla z21.h, z9.h, z1.h[0]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" + "fmla z25.h, z9.h, z2.h[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.h, z9.h, z3.h[0]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.h, z10.h, z0.h[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.h, z10.h, z1.h[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.h, z10.h, z2.h[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.h, z10.h, z3.h[0]\n" "ld1h z10.h, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.h, z11.h, z0.h[0]\n" @@ -3622,7 +3565,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z1.h[7]\n" "fmla z27.h, z15.h, z2.h[7]\n" "fmla z31.h, z15.h, z3.h[7]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3645,7 +3588,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[0]\n" "fmla z27.h, z11.h, z6.h[0]\n" "fmla z31.h, z11.h, z7.h[0]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3668,7 +3611,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[1]\n" "fmla z27.h, z15.h, z6.h[1]\n" "fmla z31.h, z15.h, z7.h[1]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3691,7 +3634,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[2]\n" "fmla z27.h, z11.h, z6.h[2]\n" "fmla z31.h, z11.h, z7.h[2]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3714,7 +3657,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[3]\n" "fmla z27.h, z15.h, z6.h[3]\n" "fmla z31.h, z15.h, z7.h[3]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" @@ -3737,7 +3680,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[4]\n" "fmla z27.h, z11.h, z6.h[4]\n" "fmla z31.h, z11.h, z7.h[4]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1h z12.h, p0/z, [%[b_ptr0]]\n" @@ -3760,7 +3703,7 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z15.h, z5.h[5]\n" "fmla z27.h, z15.h, z6.h[5]\n" "fmla z31.h, z15.h, z7.h[5]\n" - "b.eq 5f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1h z8.h, p0/z, [%[b_ptr0]]\n" "ld1h z9.h, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -3782,17 +3725,51 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld "fmla z23.h, z11.h, z5.h[6]\n" "fmla z27.h, z11.h, z6.h[6]\n" "fmla z31.h, z11.h, z7.h[6]\n" - "5:\n" + "4:\n" + "ld1rh z14.h, p7/z, [%[minptr]]\n" + "ld1rh z15.h, p7/z, [%[maxptr]]\n" + "fmax z16.h, p7/m, z16.h, z14.h\n" + "fmax z17.h, p7/m, z17.h, z14.h\n" + "fmax z18.h, p7/m, z18.h, z14.h\n" + "fmax z19.h, p7/m, z19.h, z14.h\n" + "fmin z16.h, p7/m, z16.h, z15.h\n" + "fmin z17.h, p7/m, z17.h, z15.h\n" + "fmin z18.h, p7/m, z18.h, z15.h\n" + "fmin z19.h, p7/m, z19.h, z15.h\n" "st1h z16.h, p0, [%[c_ptr0]]\n" + "fmax z20.h, p7/m, z20.h, z14.h\n" + "fmax z21.h, p7/m, z21.h, z14.h\n" + "fmax z22.h, p7/m, z22.h, z14.h\n" "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.h, p7/m, z23.h, z14.h\n" + "fmin z20.h, p7/m, z20.h, z15.h\n" + "fmin z21.h, p7/m, z21.h, z15.h\n" "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.h, p7/m, z22.h, z15.h\n" + "fmin z23.h, p7/m, z23.h, z15.h\n" + "fmax z24.h, p7/m, z24.h, z14.h\n" "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.h, p7/m, z25.h, z14.h\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.h, p7/m, z26.h, z14.h\n" "st1h z20.h, p0, [c_ptr1]\n" + "fmin z24.h, p7/m, z24.h, z15.h\n" + "fmin z25.h, p7/m, z25.h, z15.h\n" + "fmax z27.h, p7/m, z27.h, z14.h\n" "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.h, p7/m, z26.h, z15.h\n" + "fmax z28.h, p7/m, z28.h, z14.h\n" + "fmax z29.h, p7/m, z29.h, z14.h\n" "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.h, p7/m, z27.h, z15.h\n" + "fmax z30.h, p7/m, z30.h, z14.h\n" + "fmin z28.h, p7/m, z28.h, z15.h\n" "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.h, p7/m, z29.h, z15.h\n" + "fmax z31.h, p7/m, z31.h, z14.h\n" + "fmin z30.h, p7/m, z30.h, z15.h\n" "st1h z24.h, p0, [c_ptr2]\n" + "fmin z31.h, p7/m, z31.h, z15.h\n" "st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n" "st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n" "st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n" @@ -3807,11 +3784,12 @@ void sve_native_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, int ld ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp index d7f9f20074..19e5fbd974 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,13 +25,11 @@ #ifdef __ARM_FEATURE_SVE - - namespace arm_gemm { // Actual kernel implementations -void sve_native_fp32_mla_4VLx4(const float *, int, const float *, int ldb, float *, int, float, int, int, int); +void sve_native_fp32_mla_4VLx4(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); class native_fp32_mla_4VLx4 { @@ -39,10 +37,10 @@ public: typedef float operand_type; typedef float result_type; - typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, float, int, int, int); + typedef void (*kern_type)(const float *, int, const float *, int ldb, float *, int, int, int, int, const float *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -52,20 +50,32 @@ public: return get_vector_length<float>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 1; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return true; + } + + static constexpr bool supports_activation() + { + return true; + } + // Default to the generic kernel kern_type kernel=sve_native_fp32_mla_4VLx4; - native_fp32_mla_4VLx4(const CPUInfo *ci) - { - - } + native_fp32_mla_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp index 6e225669fc..3fc0e5fa36 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,30 +25,49 @@ #include <algorithm> +#include "arm_gemm.hpp" #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); +void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool append) { const long loops_count = ((K + 4) / 8) - 1; K -= loops_count * 8; const long regs_count = (K / 4) - 1; K -= (regs_count + 1) * 4; const long leftovers = K; + float nullbias[256]; + if (!append && !bias) { + memset(nullbias, 0, (4 * get_vector_length<float>() * sizeof(float))); + } + float minval = - static_cast<float>(std::numeric_limits<float>::infinity()); + float maxval = static_cast<float>(std::numeric_limits<float>::infinity()); + const float * const minptr = &minval; + const float * const maxptr = &maxval; + + switch(act.type) + { + default: + case Activation::Type::None: + break; + case Activation::Type::BoundedReLU: + maxval = static_cast<float>(act.param1); + /* fall through */ + case Activation::Type::ReLU: + minval = 0.0f; + break; + } for (int y=0; y<M; y+=4) { const float * const a_ptr0_base = A + (y * lda); const unsigned long ldab = lda * sizeof(float); float *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(float); for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>())); - const float *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -56,6 +75,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, const float *a_ptr0 = a_ptr0_base; const float *b_ptr0 = B + x0; long ldbb = ldb * sizeof(float); + const unsigned long ldcb = ldc * sizeof(float); + const float *biasptr = bias ? bias+x0 : nullbias; switch(M-y) { case 1: @@ -64,191 +85,173 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z19.s, #0\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "whilelt p2.s, %[temp], %[width]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" + "incw %[temp], all, mul #1\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "whilelt p3.s, %[temp], %[width]\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "cbz %[loops], 1f\n" "2:\n" - "cbz %[loops], 3f\n" - "4:\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "subs %[loops], %[loops], #0x1\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z16.s, z12.s, z0.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z14.s, z0.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z19.s, z15.s, z0.s[1]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" - "fmla z19.s, z11.s, z0.s[2]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z0.s[3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z13.s, z0.s[3]\n" - "fmla z18.s, z14.s, z0.s[3]\n" - "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z18.s, z14.s, z0.s[3]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "fmla z18.s, z10.s, z4.s[0]\n" - "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z18.s, z10.s, z4.s[0]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "fmla z17.s, z13.s, z4.s[1]\n" - "fmla z18.s, z14.s, z4.s[1]\n" - "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[1]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z14.s, z4.s[1]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[2]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "fmla z18.s, z10.s, z4.s[2]\n" - "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z4.s[3]\n" - "fmla z17.s, z13.s, z4.s[3]\n" - "fmla z18.s, z14.s, z4.s[3]\n" - "fmla z19.s, z15.s, z4.s[3]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[3]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z14.s, z4.s[3]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z15.s, z4.s[3]\n" + "b.ne 2b\n" + "1:\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "b.ne 4b\n" - "3:\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z18.s, z10.s, z0.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.s, z11.s, z0.s[0]\n" - "fmla z16.s, z12.s, z0.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" - "fmla z18.s, z14.s, z0.s[1]\n" + "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.s, z15.s, z0.s[1]\n" + "fmla z16.s, z12.s, z0.s[1]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z18.s, z14.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.s, z8.s, z0.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z19.s, z15.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z16.s, z8.s, z0.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z12.s, z0.s[3]\n" - "fmla z17.s, z13.s, z0.s[3]\n" - "fmla z18.s, z14.s, z0.s[3]\n" - "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[3]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z14.s, z0.s[3]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z15.s, z0.s[3]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[0]\n" "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z11.s, z4.s[0]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z16.s, z12.s, z4.s[1]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z17.s, z13.s, z4.s[1]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z15.s, z4.s[1]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "fmla z17.s, z9.s, z4.s[2]\n" - "fmla z18.s, z10.s, z4.s[2]\n" - "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z4.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z4.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z11.s, z4.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" "fmla z17.s, z13.s, z4.s[3]\n" "fmla z18.s, z14.s, z4.s[3]\n" "fmla z19.s, z15.s, z4.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -259,7 +262,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -270,7 +273,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z13.s, z0.s[1]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -280,37 +283,37 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[2]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z18.s, z10.s, z0.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z19.s, z11.s, z0.s[0]\n" - "fmla z16.s, z12.s, z0.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" - "fmla z18.s, z14.s, z0.s[1]\n" + "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.s, z15.s, z0.s[1]\n" + "fmla z16.s, z12.s, z0.s[1]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z18.s, z14.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "fmla z16.s, z8.s, z0.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" - "fmla z18.s, z10.s, z0.s[2]\n" - "fmla z19.s, z11.s, z0.s[2]\n" + "fmla z19.s, z15.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z16.s, z8.s, z0.s[2]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z18.s, z10.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z16.s, z12.s, z0.s[3]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -321,7 +324,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z4.s[0]\n" "fmla z18.s, z10.s, z4.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -332,7 +335,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z13.s, z4.s[1]\n" "fmla z18.s, z14.s, z4.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -342,14 +345,24 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z4.s[2]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -363,61 +376,33 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z20.s, #0\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z21.s, #0\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z22.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z23.s, #0\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" + "incw %[temp], all, mul #1\n" + "add a_ptr1, a_ptr1, #0x10\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "whilelt p3.s, %[temp], %[width]\n" + "mov z22.d, z18.d\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "2:\n" - "add a_ptr1, a_ptr1, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "cbz %[loops], 3f\n" - "4:\n" + "mov z23.d, z19.d\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" @@ -425,200 +410,202 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z22.s, z10.s, z1.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z23.s, z11.s, z1.s[0]\n" + "fmla z19.s, z11.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z16.s, z12.s, z0.s[1]\n" + "fmla z23.s, z11.s, z1.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z20.s, z12.s, z1.s[1]\n" + "fmla z16.s, z12.s, z0.s[1]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z17.s, z13.s, z0.s[1]\n" + "fmla z20.s, z12.s, z1.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z17.s, z13.s, z0.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z22.s, z14.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z23.s, z15.s, z1.s[1]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z16.s, z8.s, z0.s[2]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[2]\n" + "fmla z16.s, z8.s, z0.s[2]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z20.s, z8.s, z1.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z22.s, z10.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[2]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z16.s, z12.s, z0.s[3]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z12.s, z1.s[3]\n" + "fmla z16.s, z12.s, z0.s[3]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z20.s, z12.s, z1.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z22.s, z14.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "fmla z16.s, z8.s, z4.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "fmla z16.s, z8.s, z4.s[0]\n" + "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z21.s, z9.s, z5.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" "fmla z22.s, z10.s, z5.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" "fmla z22.s, z14.s, z5.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z5.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z22.s, z10.s, z5.s[2]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[3]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[3]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[3]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z5.s[3]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[3]\n" "fmla z22.s, z14.s, z5.s[3]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "b.ne 4b\n" - "3:\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" + "b.ne 2b\n" + "1:\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z22.s, z14.s, z1.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z1.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z22.s, z10.s, z1.s[2]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[3]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[3]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z13.s, z1.s[3]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" "fmla z22.s, z14.s, z1.s[3]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[3]\n" "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[0]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" "fmla z20.s, z8.s, z5.s[0]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z17.s, z9.s, z4.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z21.s, z9.s, z5.s[0]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z22.s, z10.s, z5.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z23.s, z11.s, z5.s[0]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z4.s[1]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z22.s, z14.s, z5.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z4.s[2]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z20.s, z8.s, z5.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z17.s, z9.s, z4.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" "fmla z22.s, z10.s, z5.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" @@ -631,7 +618,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z5.s[3]\n" "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -646,7 +633,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z1.s[0]\n" "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -661,7 +648,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z1.s[1]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -675,40 +662,41 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z1.s[2]\n" "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z22.s, z10.s, z1.s[0]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z21.s, z13.s, z1.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" "fmla z22.s, z14.s, z1.s[1]\n" + "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z8.s, z0.s[2]\n" - "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z20.s, z8.s, z1.s[2]\n" - "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z17.s, z9.s, z0.s[2]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z18.s, z10.s, z0.s[2]\n" "fmla z22.s, z10.s, z1.s[2]\n" @@ -722,7 +710,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z1.s[3]\n" "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -737,7 +725,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z5.s[0]\n" "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -752,7 +740,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z14.s, z5.s[1]\n" "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -766,10 +754,28 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z22.s, z10.s, z5.s[2]\n" "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" "st1w z20.s, p0, [c_ptr1]\n" @@ -779,7 +785,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -791,85 +797,45 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "c_ptr2 .req X3\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "whilelt p6.s, %[temp], %[leftovers]\n" "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "whilelt p2.s, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z21.s, #0\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z22.s, #0\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z23.s, #0\n" + "incw %[temp], all, mul #1\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z24.s, #0\n" - "add a_ptr1, a_ptr1, #0x10\n" - "mov z25.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z26.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z27.s, #0\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" + "whilelt p3.s, %[temp], %[width]\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" + "mov z26.d, z18.d\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" + "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "add a_ptr2, a_ptr2, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "2:\n" + "mov z23.d, z19.d\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" @@ -879,28 +845,28 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z6.s, p7/z, [a_ptr2]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z25.s, z9.s, z2.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z18.s, z10.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z26.s, z10.s, z2.s[0]\n" + "fmla z22.s, z10.s, z1.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "fmla z19.s, z11.s, z0.s[0]\n" + "fmla z26.s, z10.s, z2.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" + "fmla z19.s, z11.s, z0.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z20.s, z12.s, z1.s[1]\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z12.s, z2.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -916,9 +882,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -933,9 +899,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[3]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" @@ -953,12 +919,12 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z21.s, z9.s, z5.s[0]\n" "fmla z25.s, z9.s, z6.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z18.s, z10.s, z4.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z4.s[0]\n" "fmla z22.s, z10.s, z5.s[0]\n" "fmla z26.s, z10.s, z6.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -970,9 +936,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z6.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" @@ -987,9 +953,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z6.s[2]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[2]\n" @@ -1004,9 +970,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[3]\n" "fmla z24.s, z12.s, z6.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[3]\n" "fmla z21.s, z13.s, z5.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z6.s[3]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[3]\n" @@ -1016,37 +982,37 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" + "b.ne 2b\n" + "1:\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p7/z, [a_ptr2]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z22.s, z10.s, z1.s[0]\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" @@ -1061,9 +1027,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[2]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[2]\n" @@ -1078,9 +1044,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[3]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[3]\n" @@ -1098,12 +1064,15 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z21.s, z9.s, z5.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z18.s, z10.s, z4.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z18.s, z10.s, z4.s[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z22.s, z10.s, z5.s[0]\n" "fmla z26.s, z10.s, z6.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" @@ -1115,9 +1084,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z6.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z4.s[1]\n" @@ -1152,7 +1121,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z4.s[3]\n" "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1171,7 +1140,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z0.s[0]\n" "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -1190,7 +1159,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z0.s[1]\n" "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1208,36 +1177,38 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z0.s[2]\n" "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p6/z, [a_ptr2]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z22.s, z10.s, z1.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z23.s, z11.s, z1.s[0]\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z27.s, z11.s, z2.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z16.s, z12.s, z0.s[1]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" + "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z13.s, z2.s[1]\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z14.s, z0.s[1]\n" @@ -1272,7 +1243,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z0.s[3]\n" "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1291,7 +1262,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z4.s[0]\n" "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -1310,7 +1281,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z15.s, z4.s[1]\n" "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1328,14 +1299,40 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z19.s, z11.s, z4.s[2]\n" "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" "st1w z24.s, p0, [c_ptr2]\n" @@ -1347,7 +1344,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1362,103 +1359,53 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "c_ptr3 .req X5\n" "add a_ptr1, %[a_ptr0], %[lda]\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[leftovers]\n" - "whilelt p0.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" "add a_ptr2, a_ptr1, %[lda]\n" "add c_ptr2, c_ptr1, %[ldc]\n" - "ptrue p7.s\n" - "whilelt p1.s, %[temp], %[width]\n" "add a_ptr3, a_ptr2, %[lda]\n" "add c_ptr3, c_ptr2, %[ldc]\n" + "whilelt p6.s, %[temp], %[leftovers]\n" + "whilelt p0.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" + "ptrue p7.s\n" + "ld1w z16.s, p0/z, [%[biasptr]]\n" + "whilelt p1.s, %[temp], %[width]\n" "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" + "incw %[temp], all, mul #1\n" + "mov z20.d, z16.d\n" + "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "mov z24.d, z16.d\n" "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z28.d, z16.d\n" "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "mov z20.s, #0\n" + "whilelt p2.s, %[temp], %[width]\n" + "mov z21.d, z17.d\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "mov z21.s, #0\n" + "mov z25.d, z17.d\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "mov z22.s, #0\n" + "mov z29.d, z17.d\n" + "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "mov z23.s, #0\n" - "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "mov z24.s, #0\n" + "incw %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z25.s, #0\n" + "mov z22.d, z18.d\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z26.s, #0\n" + "mov z26.d, z18.d\n" + "whilelt p3.s, %[temp], %[width]\n" + "mov z30.d, z18.d\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z27.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" - "mov z28.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 2f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "fmul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "fmul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "fmul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "fmul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "fmul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "fmul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "fmul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "fmul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z28.s, p0/z, [c_ptr3]\n" - "fmul z24.s, p7/m, z24.s, z15.s\n" - "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "fmul z25.s, p7/m, z25.s, z15.s\n" - "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "fmul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "fmul z27.s, p7/m, z27.s, z15.s\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "fmul z28.s, p7/m, z28.s, z15.s\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "fmul z29.s, p7/m, z29.s, z15.s\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "fmul z30.s, p7/m, z30.s, z15.s\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmul z31.s, p7/m, z31.s, z15.s\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" + "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" - "add a_ptr3, a_ptr3, #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "2:\n" + "mov z23.d, z19.d\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "mov z27.d, z19.d\n" "ld1w z13.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "mov z31.d, z19.d\n" "ld1w z14.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 3f\n" - "4:\n" + "cbz %[loops], 1f\n" + "2:\n" "fmla z16.s, z8.s, z0.s[0]\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" "fmla z20.s, z8.s, z1.s[0]\n" @@ -1470,23 +1417,23 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z17.s, z9.s, z0.s[0]\n" "ld1rqw z7.s, p7/z, [a_ptr3]\n" "fmla z21.s, z9.s, z1.s[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "subs %[loops], %[loops], #0x1\n" "fmla z25.s, z9.s, z2.s[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z29.s, z9.s, z3.s[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "fmla z22.s, z10.s, z1.s[0]\n" + "fmla z18.s, z10.s, z0.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "fmla z22.s, z10.s, z1.s[0]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "fmla z26.s, z10.s, z2.s[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" + "add a_ptr1, a_ptr1, #0x20\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" + "add a_ptr2, a_ptr2, #0x20\n" "fmla z23.s, z11.s, z1.s[0]\n" - "subs %[loops], %[loops], #0x1\n" + "add a_ptr3, a_ptr3, #0x20\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" "ld1w z11.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1495,8 +1442,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "fmla z29.s, z13.s, z3.s[1]\n" @@ -1516,8 +1463,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1537,8 +1484,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" "fmla z28.s, z12.s, z3.s[3]\n" - "fmla z17.s, z13.s, z0.s[3]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" "fmla z25.s, z13.s, z2.s[3]\n" "fmla z29.s, z13.s, z3.s[3]\n" @@ -1562,9 +1509,9 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" "fmla z28.s, z8.s, z7.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" "fmla z21.s, z9.s, z5.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z6.s[0]\n" "fmla z29.s, z9.s, z7.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1583,8 +1530,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z28.s, z12.s, z7.s[1]\n" - "fmla z17.s, z13.s, z4.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" "fmla z25.s, z13.s, z6.s[1]\n" "fmla z29.s, z13.s, z7.s[1]\n" @@ -1604,8 +1551,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[2]\n" "fmla z24.s, z8.s, z6.s[2]\n" "fmla z28.s, z8.s, z7.s[2]\n" - "fmla z17.s, z9.s, z4.s[2]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z4.s[2]\n" "fmla z21.s, z9.s, z5.s[2]\n" "fmla z25.s, z9.s, z6.s[2]\n" "fmla z29.s, z9.s, z7.s[2]\n" @@ -1625,8 +1572,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z5.s[3]\n" "fmla z24.s, z12.s, z6.s[3]\n" "fmla z28.s, z12.s, z7.s[3]\n" - "fmla z17.s, z13.s, z4.s[3]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[3]\n" "fmla z21.s, z13.s, z5.s[3]\n" "fmla z25.s, z13.s, z6.s[3]\n" "fmla z29.s, z13.s, z7.s[3]\n" @@ -1640,28 +1587,28 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "b.ne 4b\n" - "3:\n" - "cbz %[regs], 5f\n" - "fmla z16.s, z8.s, z0.s[0]\n" + "b.ne 2b\n" + "1:\n" "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" + "cbz %[regs], 3f\n" + "fmla z16.s, z8.s, z0.s[0]\n" "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p7/z, [a_ptr1]\n" - "fmla z28.s, z8.s, z3.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p7/z, [a_ptr2]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z28.s, z8.s, z3.s[0]\n" "ld1rqw z7.s, p7/z, [a_ptr3]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" "fmla z29.s, z9.s, z3.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" "fmla z22.s, z10.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -1674,8 +1621,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "fmla z29.s, z13.s, z3.s[1]\n" @@ -1695,8 +1642,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z1.s[2]\n" "fmla z24.s, z8.s, z2.s[2]\n" "fmla z28.s, z8.s, z3.s[2]\n" - "fmla z17.s, z9.s, z0.s[2]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z9.s, z0.s[2]\n" "fmla z21.s, z9.s, z1.s[2]\n" "fmla z25.s, z9.s, z2.s[2]\n" "fmla z29.s, z9.s, z3.s[2]\n" @@ -1716,8 +1663,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[3]\n" "fmla z24.s, z12.s, z2.s[3]\n" "fmla z28.s, z12.s, z3.s[3]\n" - "fmla z17.s, z13.s, z0.s[3]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[3]\n" "fmla z21.s, z13.s, z1.s[3]\n" "fmla z25.s, z13.s, z2.s[3]\n" "fmla z29.s, z13.s, z3.s[3]\n" @@ -1740,11 +1687,15 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z8.s, z5.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "fmla z24.s, z8.s, z6.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "fmla z28.s, z8.s, z7.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z17.s, z9.s, z4.s[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "fmla z21.s, z9.s, z5.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "addvl a_ptr2, a_ptr2, #2\n" "fmla z25.s, z9.s, z6.s[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "fmla z29.s, z9.s, z7.s[0]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z4.s[0]\n" @@ -1762,8 +1713,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z5.s[1]\n" "fmla z24.s, z12.s, z6.s[1]\n" "fmla z28.s, z12.s, z7.s[1]\n" - "fmla z17.s, z13.s, z4.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z4.s[1]\n" "fmla z21.s, z13.s, z5.s[1]\n" "fmla z25.s, z13.s, z6.s[1]\n" "fmla z29.s, z13.s, z7.s[1]\n" @@ -1810,7 +1761,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z5.s[3]\n" "fmla z27.s, z15.s, z6.s[3]\n" "fmla z31.s, z15.s, z7.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1833,7 +1784,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z1.s[0]\n" "fmla z27.s, z11.s, z2.s[0]\n" "fmla z31.s, z11.s, z3.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -1856,7 +1807,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z1.s[1]\n" "fmla z27.s, z15.s, z2.s[1]\n" "fmla z31.s, z15.s, z3.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1878,27 +1829,30 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z1.s[2]\n" "fmla z27.s, z11.s, z2.s[2]\n" "fmla z31.s, z11.s, z3.s[2]\n" - "b 6f\n" - "5:\n" + "b 4f\n" + "3:\n" "fmla z16.s, z8.s, z0.s[0]\n" - "ld1w z15.s, p3/z, [%[b_ptr0], #3, MUL VL]\n" - "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n" - "fmla z24.s, z8.s, z2.s[0]\n" + "fmla z20.s, z8.s, z1.s[0]\n" "ld1rqw z5.s, p6/z, [a_ptr1]\n" - "fmla z28.s, z8.s, z3.s[0]\n" + "fmla z24.s, z8.s, z2.s[0]\n" "ld1rqw z6.s, p6/z, [a_ptr2]\n" - "fmla z17.s, z9.s, z0.s[0]\n" + "fmla z28.s, z8.s, z3.s[0]\n" "ld1rqw z7.s, p6/z, [a_ptr3]\n" - "fmla z21.s, z9.s, z1.s[0]\n" + "fmla z17.s, z9.s, z0.s[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "fmla z21.s, z9.s, z1.s[0]\n" + "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "fmla z25.s, z9.s, z2.s[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "fmla z29.s, z9.s, z3.s[0]\n" + "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" "fmla z18.s, z10.s, z0.s[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "fmla z22.s, z10.s, z1.s[0]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" + "addvl a_ptr2, a_ptr2, #1\n" "fmla z26.s, z10.s, z2.s[0]\n" - "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" + "addvl a_ptr3, a_ptr3, #1\n" "fmla z30.s, z10.s, z3.s[0]\n" "ld1w z10.s, p2/z, [%[b_ptr0], #2, MUL VL]\n" "fmla z19.s, z11.s, z0.s[0]\n" @@ -1911,8 +1865,8 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z20.s, z12.s, z1.s[1]\n" "fmla z24.s, z12.s, z2.s[1]\n" "fmla z28.s, z12.s, z3.s[1]\n" - "fmla z17.s, z13.s, z0.s[1]\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" + "fmla z17.s, z13.s, z0.s[1]\n" "fmla z21.s, z13.s, z1.s[1]\n" "fmla z25.s, z13.s, z2.s[1]\n" "fmla z29.s, z13.s, z3.s[1]\n" @@ -1959,7 +1913,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z1.s[3]\n" "fmla z27.s, z15.s, z2.s[3]\n" "fmla z31.s, z15.s, z3.s[3]\n" - "cbz %[blocks], 6f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" @@ -1982,7 +1936,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z5.s[0]\n" "fmla z27.s, z11.s, z6.s[0]\n" "fmla z31.s, z11.s, z7.s[0]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "subs %[blocks], %[blocks], #0x1\n" "ld1w z12.s, p0/z, [%[b_ptr0]]\n" @@ -2005,7 +1959,7 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z15.s, z5.s[1]\n" "fmla z27.s, z15.s, z6.s[1]\n" "fmla z31.s, z15.s, z7.s[1]\n" - "b.eq 6f\n" + "b.eq 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "ld1w z8.s, p0/z, [%[b_ptr0]]\n" "ld1w z9.s, p1/z, [%[b_ptr0], #1, MUL VL]\n" @@ -2027,17 +1981,51 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, "fmla z23.s, z11.s, z5.s[2]\n" "fmla z27.s, z11.s, z6.s[2]\n" "fmla z31.s, z11.s, z7.s[2]\n" - "6:\n" + "4:\n" + "ld1rw z14.s, p7/z, [%[minptr]]\n" + "ld1rw z15.s, p7/z, [%[maxptr]]\n" + "fmax z16.s, p7/m, z16.s, z14.s\n" + "fmax z17.s, p7/m, z17.s, z14.s\n" + "fmax z18.s, p7/m, z18.s, z14.s\n" + "fmax z19.s, p7/m, z19.s, z14.s\n" + "fmin z16.s, p7/m, z16.s, z15.s\n" + "fmin z17.s, p7/m, z17.s, z15.s\n" + "fmin z18.s, p7/m, z18.s, z15.s\n" + "fmin z19.s, p7/m, z19.s, z15.s\n" "st1w z16.s, p0, [%[c_ptr0]]\n" + "fmax z20.s, p7/m, z20.s, z14.s\n" + "fmax z21.s, p7/m, z21.s, z14.s\n" + "fmax z22.s, p7/m, z22.s, z14.s\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" + "fmax z23.s, p7/m, z23.s, z14.s\n" + "fmin z20.s, p7/m, z20.s, z15.s\n" + "fmin z21.s, p7/m, z21.s, z15.s\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" + "fmin z22.s, p7/m, z22.s, z15.s\n" + "fmin z23.s, p7/m, z23.s, z15.s\n" + "fmax z24.s, p7/m, z24.s, z14.s\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" + "fmax z25.s, p7/m, z25.s, z14.s\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" + "fmax z26.s, p7/m, z26.s, z14.s\n" "st1w z20.s, p0, [c_ptr1]\n" + "fmin z24.s, p7/m, z24.s, z15.s\n" + "fmin z25.s, p7/m, z25.s, z15.s\n" + "fmax z27.s, p7/m, z27.s, z14.s\n" "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n" + "fmin z26.s, p7/m, z26.s, z15.s\n" + "fmax z28.s, p7/m, z28.s, z14.s\n" + "fmax z29.s, p7/m, z29.s, z14.s\n" "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n" + "fmin z27.s, p7/m, z27.s, z15.s\n" + "fmax z30.s, p7/m, z30.s, z14.s\n" + "fmin z28.s, p7/m, z28.s, z15.s\n" "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n" + "fmin z29.s, p7/m, z29.s, z15.s\n" + "fmax z31.s, p7/m, z31.s, z14.s\n" + "fmin z30.s, p7/m, z30.s, z15.s\n" "st1w z24.s, p0, [c_ptr2]\n" + "fmin z31.s, p7/m, z31.s, z15.s\n" "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n" "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n" "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n" @@ -2052,11 +2040,12 @@ void sve_native_fp32_mla_4VLx4(const float *A, int lda, const float *B, int ldb, ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [leftovers] "r" (leftovers), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp index 8b98358cd4..1b9d1312b5 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,11 +27,12 @@ #include <cstdint> + namespace arm_gemm { // Actual kernel implementations -void sve_native_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int); +void sve_native_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class native_s8s32_dot_4VLx4 { @@ -39,10 +40,10 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int ldb, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -52,20 +53,32 @@ public: return get_vector_length<int32_t>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + // Default to the generic kernel kern_type kernel=sve_native_s8s32_dot_4VLx4; - native_s8s32_dot_4VLx4(const CPUInfo *ci) - { - - } + native_s8s32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp index abee1bbe1f..95cf88ab54 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp @@ -25,14 +25,14 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int ldb, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0); +void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int ldb, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; const long regs_count = (K / 16) - 1; @@ -46,11 +46,9 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l const unsigned long ldab = lda * sizeof(int8_t); int32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(int32_t); for (int x0=0; x0<N; x0+=(4 * get_vector_length<int32_t>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<int32_t>())); - const int32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -62,95 +60,68 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l const int8_t *b_ptr2 = b_ptr1 + ldb; const int8_t *b_ptr3 = b_ptr2 + ldb; long ldbb = ldb * sizeof(int8_t) * 4; + const unsigned long ldcb = ldc * sizeof(int32_t); switch(M-y) { case 1: __asm __volatile ( + "mov z16.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z17.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z18.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z19.s, #0\n" "incw %[temp], all, mul #1\n" "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z19.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "whilelt p1.s, %[temp], %[width]\n" + "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip2 z11.b, z8.b, z9.b\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" + "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "whilelt p3.s, %[temp], %[width]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z10.b, z11.b, z12.b\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "sdot z16.s, z8.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "zip2 z15.b, z12.b, z13.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "zip2 z15.b, z12.b, z13.b\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" "sdot z18.s, z10.b, z0.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z12.b, z13.b, z14.b\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" @@ -289,32 +260,32 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "b.ne 3b\n" - "2:\n" "zip1 z10.b, z10.b, z8.b\n" "zip1 z8.b, z9.b, z10.b\n" + "b.ne 2b\n" + "1:\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "sdot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" @@ -377,6 +348,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z10.b, z10.b, z8.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z8.b, z9.b, z10.b\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" @@ -437,7 +409,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[3]\n" "sdot z18.s, z14.b, z4.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -459,7 +431,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[0]\n" "sdot z18.s, z10.b, z0.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -481,7 +453,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[1]\n" "sdot z18.s, z14.b, z0.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -502,31 +474,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[2]\n" "sdot z18.s, z10.b, z0.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -540,33 +512,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -580,33 +552,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[2]\n" "sdot z18.s, z10.b, z0.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -620,33 +592,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[1]\n" "sdot z18.s, z14.b, z0.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -660,36 +632,36 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z0.b[0]\n" "sdot z18.s, z10.b, z0.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "sdot z16.s, z12.b, z0.b[1]\n" + "sdot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z17.s, z13.b, z0.b[1]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "sdot z16.s, z12.b, z0.b[1]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "sdot z17.s, z13.b, z0.b[1]\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z14.b, z0.b[1]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z15.b, z0.b[1]\n" @@ -718,7 +690,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z0.b[3]\n" "sdot z18.s, z14.b, z0.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -740,7 +712,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[0]\n" "sdot z18.s, z10.b, z4.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -762,7 +734,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[1]\n" "sdot z18.s, z14.b, z4.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -783,31 +755,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[2]\n" "sdot z18.s, z10.b, z4.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -821,33 +793,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[3]\n" "sdot z18.s, z14.b, z4.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -861,33 +833,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[2]\n" "sdot z18.s, z10.b, z4.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -901,33 +873,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z13.b, z4.b[1]\n" "sdot z18.s, z14.b, z4.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -941,14 +913,14 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z17.s, z9.b, z4.b[0]\n" "sdot z18.s, z10.b, z4.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -956,125 +928,81 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l __asm __volatile ( "a_ptr1 .req X0\n" "c_ptr1 .req X1\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" + "mov z17.s, #0\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z18.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z19.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z20.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z21.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" + "mov z23.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z20.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z21.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "whilelt p1.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "ptrue p7.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z22.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z23.s, #0\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "ld1rqb z1.b, p7/z, [a_ptr1]\n" "zip1 z10.b, z10.b, z8.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "zip1 z10.b, z11.b, z12.b\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "sdot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "sdot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "sdot z20.s, z8.b, z1.b[0]\n" "subs %[loops], %[loops], #0x1\n" - "sdot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "sdot z21.s, z9.b, z1.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "sdot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -1244,38 +1172,38 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "sdot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "sdot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1357,23 +1285,25 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z20.s, z8.b, z5.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z14.b, z14.b, z12.b\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z4.b[0]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z18.s, z10.b, z4.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z4.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z23.s, z11.b, z5.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "sdot z23.s, z11.b, z5.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1420,7 +1350,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1446,7 +1376,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1472,7 +1402,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1497,31 +1427,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1539,33 +1469,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1583,33 +1513,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[2]\n" "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1627,33 +1557,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[1]\n" "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1671,35 +1601,36 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z1.b[0]\n" "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "sdot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "sdot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "sdot z19.s, z11.b, z0.b[0]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "sdot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1746,7 +1677,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z1.b[3]\n" "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1772,7 +1703,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1798,7 +1729,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1823,31 +1754,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1865,33 +1796,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[3]\n" "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1909,33 +1840,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[2]\n" "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1953,33 +1884,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z14.b, z5.b[1]\n" "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1997,7 +1928,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z22.s, z10.b, z5.b[0]\n" "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -2010,7 +1941,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -2020,119 +1951,65 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "a_ptr2 .req X1\n" "c_ptr1 .req X2\n" "c_ptr2 .req X3\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z17.s, #0\n" "add a_ptr2, a_ptr1, %[lda]\n" + "mov z18.s, #0\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z19.s, #0\n" "add c_ptr2, c_ptr1, %[ldc]\n" + "mov z20.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z21.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z22.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z23.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z24.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z25.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z21.s, #0\n" + "mov z26.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z22.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "incw %[temp], all, mul #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z23.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z24.s, #0\n" + "ptrue p7.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "whilelt p2.s, %[temp], %[width]\n" "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "mov z25.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "mov z26.s, #0\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "mov z27.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "incw %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z8.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z9.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "zip2 z11.b, z11.b, z12.b\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "sdot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "sdot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "sdot z24.s, z8.b, z2.b[0]\n" @@ -2361,24 +2238,23 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "sdot z21.s, z9.b, z1.b[0]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" @@ -2386,12 +2262,13 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" "sdot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" @@ -2491,8 +2368,11 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip1 z14.b, z14.b, z12.b\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" @@ -2571,7 +2451,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2601,7 +2481,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -2631,7 +2511,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2660,31 +2540,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2706,33 +2586,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2754,33 +2634,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[2]\n" "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2802,33 +2682,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[1]\n" "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2850,34 +2730,36 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z0.b[0]\n" "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "sdot z21.s, z9.b, z1.b[0]\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "sdot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" @@ -2942,7 +2824,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z0.b[3]\n" "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2972,7 +2854,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3002,7 +2884,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3031,31 +2913,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3077,33 +2959,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[3]\n" "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3125,33 +3007,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[2]\n" "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3173,33 +3055,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z15.b, z4.b[1]\n" "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3221,7 +3103,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z19.s, z11.b, z4.b[0]\n" "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -3240,7 +3122,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -3253,175 +3135,109 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "c_ptr1 .req X3\n" "c_ptr2 .req X4\n" "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "whilelt p6.b, %[temp], %[leftovers]\n" - "whilelt p0.s, %[temp], %[width]\n" - "whilelt p4.b, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" + "add a_ptr2, a_ptr1, %[lda]\n" "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" + "add a_ptr3, a_ptr2, %[lda]\n" "mov z19.s, #0\n" - "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "mov z20.s, #0\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "mov z21.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "mov z22.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "whilelt p6.b, %[temp], %[leftovers]\n" "mov z23.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "whilelt p0.s, %[temp], %[width]\n" "mov z24.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" + "whilelt p4.b, %[temp], %[width]\n" "mov z25.s, #0\n" - "add a_ptr3, a_ptr3, #0x10\n" - "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "incw %[temp], all, mul #1\n" "mov z26.s, #0\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "mov z27.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip2 z9.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "mov z28.s, #0\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "mov z29.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" + "zip2 z11.b, z8.b, z9.b\n" + "incw %[temp], all, mul #1\n" + "zip1 z9.b, z8.b, z9.b\n" + "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "mov z30.s, #0\n" - "zip2 z15.b, z12.b, z13.b\n" - "zip1 z13.b, z12.b, z13.b\n" + "ptrue p7.b\n" "mov z31.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" - "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" - "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "zip2 z12.b, z10.b, z8.b\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "incw %[temp], all, mul #1\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "zip1 z8.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip2 z9.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z11.b, z11.b, z12.b\n" "add a_ptr3, a_ptr3, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[loops], 2f\n" - "3:\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "sdot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "sdot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1rqb z6.b, p7/z, [a_ptr2]\n" "sdot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z7.b, p7/z, [a_ptr3]\n" "sdot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p7/z, [a_ptr2]\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z8.b, z14.b, z12.b\n" - "ld1rqb z7.b, p7/z, [a_ptr3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "sdot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add a_ptr2, a_ptr2, #0x20\n" "sdot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z22.s, z10.b, z1.b[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "add a_ptr3, a_ptr3, #0x20\n" "sdot z26.s, z10.b, z2.b[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" "sdot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "sdot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "sdot z23.s, z11.b, z1.b[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -3649,29 +3465,29 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "sdot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "sdot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "sdot z28.s, z8.b, z3.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "sdot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p7/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3797,9 +3613,13 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "sdot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "sdot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "sdot z28.s, z8.b, z7.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr3, a_ptr3, #2\n" "zip1 z14.b, z14.b, z12.b\n" "sdot z17.s, z9.b, z4.b[0]\n" "sdot z21.s, z9.b, z5.b[0]\n" @@ -3893,7 +3713,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3927,7 +3747,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3961,7 +3781,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" "sdot z31.s, z15.b, z3.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3994,31 +3814,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" "sdot z31.s, z11.b, z3.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4044,33 +3864,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" "sdot z31.s, z15.b, z3.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4096,33 +3916,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[2]\n" "sdot z27.s, z11.b, z2.b[2]\n" "sdot z31.s, z11.b, z3.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4148,33 +3968,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[1]\n" "sdot z27.s, z15.b, z2.b[1]\n" "sdot z31.s, z15.b, z3.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4200,37 +4020,40 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z1.b[0]\n" "sdot z27.s, z11.b, z2.b[0]\n" "sdot z31.s, z11.b, z3.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "sdot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "sdot z24.s, z8.b, z2.b[0]\n" + "sdot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "sdot z28.s, z8.b, z3.b[0]\n" + "sdot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "sdot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p6/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "sdot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "sdot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "sdot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "sdot z25.s, z9.b, z2.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "sdot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "sdot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "sdot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "sdot z26.s, z10.b, z2.b[0]\n" "sdot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" @@ -4309,7 +4132,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z1.b[3]\n" "sdot z27.s, z15.b, z2.b[3]\n" "sdot z31.s, z15.b, z3.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4343,7 +4166,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" "sdot z31.s, z11.b, z7.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -4377,7 +4200,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" "sdot z31.s, z15.b, z7.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4410,31 +4233,31 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" "sdot z31.s, z11.b, z7.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4460,33 +4283,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[3]\n" "sdot z27.s, z15.b, z6.b[3]\n" "sdot z31.s, z15.b, z7.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4512,33 +4335,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[2]\n" "sdot z27.s, z11.b, z6.b[2]\n" "sdot z31.s, z11.b, z7.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4564,33 +4387,33 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z15.b, z5.b[1]\n" "sdot z27.s, z15.b, z6.b[1]\n" "sdot z31.s, z15.b, z7.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4616,7 +4439,7 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l "sdot z23.s, z11.b, z5.b[0]\n" "sdot z27.s, z11.b, z6.b[0]\n" "sdot z31.s, z11.b, z7.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -4641,11 +4464,12 @@ void sve_native_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int l ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp index bcbd3d35f5..33e3ac6c23 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_native_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int); +void sve_native_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class native_u8u32_dot_4VLx4 { @@ -40,10 +40,10 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, int ldb, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ - static unsigned int out_height() + static constexpr unsigned int out_height() { return 4; } @@ -53,20 +53,32 @@ public: return get_vector_length<uint32_t>() * 4; } - static unsigned int k_unroll() + static constexpr unsigned int k_unroll() { return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + // Default to the generic kernel kern_type kernel=sve_native_u8u32_dot_4VLx4; - native_u8u32_dot_4VLx4(const CPUInfo *ci) - { - - } + native_u8u32_dot_4VLx4(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp index cdcea59c5e..994d608bf6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_native_u8u32_dot_4VLx4/generic.cpp @@ -25,14 +25,14 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ldb, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ldb, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; const long regs_count = (K / 16) - 1; @@ -46,11 +46,9 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int const unsigned long ldab = lda * sizeof(uint8_t); uint32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(uint32_t); for (int x0=0; x0<N; x0+=(4 * get_vector_length<uint32_t>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<uint32_t>())); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; @@ -62,95 +60,68 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int const uint8_t *b_ptr2 = b_ptr1 + ldb; const uint8_t *b_ptr3 = b_ptr2 + ldb; long ldbb = ldb * sizeof(uint8_t) * 4; + const unsigned long ldcb = ldc * sizeof(uint32_t); switch(M-y) { case 1: __asm __volatile ( + "mov z16.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z17.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z18.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z19.s, #0\n" "incw %[temp], all, mul #1\n" "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z18.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z19.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "whilelt p1.s, %[temp], %[width]\n" + "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip2 z11.b, z8.b, z9.b\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" + "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "whilelt p3.s, %[temp], %[width]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z10.b, z11.b, z12.b\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "udot z16.s, z8.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "zip2 z15.b, z12.b, z13.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "zip2 z15.b, z12.b, z13.b\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" "udot z18.s, z10.b, z0.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "udot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z12.b, z13.b, z14.b\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" @@ -289,32 +260,32 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "b.ne 3b\n" - "2:\n" "zip1 z10.b, z10.b, z8.b\n" "zip1 z8.b, z9.b, z10.b\n" + "b.ne 2b\n" + "1:\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "udot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" @@ -377,6 +348,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "zip2 z12.b, z10.b, z8.b\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z10.b, z10.b, z8.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z8.b, z9.b, z10.b\n" "zip2 z9.b, z9.b, z10.b\n" "zip1 z10.b, z11.b, z12.b\n" @@ -437,7 +409,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -459,7 +431,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -481,7 +453,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -502,31 +474,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -540,33 +512,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -580,33 +552,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -620,33 +592,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -660,36 +632,36 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z17.s, z9.b, z0.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "udot z16.s, z12.b, z0.b[1]\n" + "udot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z17.s, z13.b, z0.b[1]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "udot z16.s, z12.b, z0.b[1]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "udot z17.s, z13.b, z0.b[1]\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z14.b, z0.b[1]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z15.b, z0.b[1]\n" @@ -718,7 +690,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -740,7 +712,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -762,7 +734,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -783,31 +755,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -821,33 +793,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -861,33 +833,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -901,33 +873,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -941,14 +913,14 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -956,125 +928,81 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int __asm __volatile ( "a_ptr1 .req X0\n" "c_ptr1 .req X1\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" + "mov z17.s, #0\n" "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z18.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z19.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z20.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z21.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" + "mov z22.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z19.s, #0\n" + "mov z23.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z20.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z21.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "whilelt p1.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "ptrue p7.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z22.s, #0\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "mov z23.s, #0\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "incw %[temp], all, mul #1\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z12.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "ld1rqb z1.b, p7/z, [a_ptr1]\n" "zip1 z10.b, z10.b, z8.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z8.b, z9.b, z10.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z9.b, z9.b, z10.b\n" + "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "zip1 z10.b, z11.b, z12.b\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z11.b, z12.b\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "udot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "udot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "udot z20.s, z8.b, z1.b[0]\n" "subs %[loops], %[loops], #0x1\n" - "udot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "udot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -1244,38 +1172,38 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "udot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "udot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1357,23 +1285,25 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z20.s, z8.b, z5.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "zip1 z14.b, z14.b, z12.b\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z4.b[0]\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z18.s, z10.b, z4.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z4.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z23.s, z11.b, z5.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "udot z23.s, z11.b, z5.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1420,7 +1350,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1446,7 +1376,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1472,7 +1402,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1497,31 +1427,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1539,33 +1469,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1583,33 +1513,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1627,33 +1557,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1671,35 +1601,36 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z1.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "udot z21.s, z9.b, z1.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip2 z8.b, z14.b, z12.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "udot z18.s, z10.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "udot z19.s, z11.b, z0.b[0]\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" @@ -1746,7 +1677,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z1.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1772,7 +1703,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -1798,7 +1729,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -1823,31 +1754,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1865,33 +1796,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[3]\n" "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1909,33 +1840,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -1953,33 +1884,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z14.b, z5.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -1997,7 +1928,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z22.s, z10.b, z5.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -2010,7 +1941,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -2020,119 +1951,65 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "a_ptr2 .req X1\n" "c_ptr1 .req X2\n" "c_ptr2 .req X3\n" + "mov z16.s, #0\n" "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z17.s, #0\n" "add a_ptr2, a_ptr1, %[lda]\n" + "mov z18.s, #0\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "mov z19.s, #0\n" "add c_ptr2, c_ptr1, %[ldc]\n" + "mov z20.s, #0\n" "whilelt p6.b, %[temp], %[leftovers]\n" + "mov z21.s, #0\n" "whilelt p0.s, %[temp], %[width]\n" + "mov z22.s, #0\n" "whilelt p4.b, %[temp], %[width]\n" + "mov z23.s, #0\n" "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" - "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mov z19.s, #0\n" + "mov z24.s, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "mov z20.s, #0\n" + "mov z25.s, #0\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" - "mov z21.s, #0\n" + "mov z26.s, #0\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "mov z22.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" + "incw %[temp], all, mul #1\n" "zip1 z9.b, z8.b, z9.b\n" "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "mov z23.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" - "mov z24.s, #0\n" + "ptrue p7.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "whilelt p2.s, %[temp], %[width]\n" "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "mov z25.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "mov z26.s, #0\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip1 z8.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "mov z27.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "incw %[temp], all, mul #1\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z8.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z9.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "zip2 z11.b, z11.b, z12.b\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "cbz %[loops], 2f\n" - "3:\n" - "udot z16.s, z8.b, z0.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "udot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "udot z24.s, z8.b, z2.b[0]\n" @@ -2361,24 +2238,23 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" - "b.ne 3b\n" - "2:\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "udot z21.s, z9.b, z1.b[0]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" @@ -2386,12 +2262,13 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -2491,8 +2368,11 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip1 z14.b, z14.b, z12.b\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -2571,7 +2451,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2601,7 +2481,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -2631,7 +2511,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2660,31 +2540,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2706,33 +2586,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2754,33 +2634,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -2802,33 +2682,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -2850,34 +2730,36 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "udot z21.s, z9.b, z1.b[0]\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "udot z21.s, z9.b, z1.b[0]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z22.s, z10.b, z1.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -2942,7 +2824,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -2972,7 +2854,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3002,7 +2884,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3031,31 +2913,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3077,33 +2959,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3125,33 +3007,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -3173,33 +3055,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -3221,7 +3103,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -3240,7 +3122,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -3253,175 +3135,109 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "c_ptr1 .req X3\n" "c_ptr2 .req X4\n" "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "whilelt p6.b, %[temp], %[leftovers]\n" - "whilelt p0.s, %[temp], %[width]\n" - "whilelt p4.b, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "ptrue p7.b\n" - "whilelt p1.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p2.s, %[temp], %[width]\n" - "incw %[temp], all, mul #1\n" - "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" "mov z16.s, #0\n" - "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" "mov z17.s, #0\n" - "ld1rqb z1.b, p7/z, [a_ptr1]\n" + "add a_ptr2, a_ptr1, %[lda]\n" "mov z18.s, #0\n" - "ld1rqb z2.b, p7/z, [a_ptr2]\n" + "add a_ptr3, a_ptr2, %[lda]\n" "mov z19.s, #0\n" - "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" "mov z20.s, #0\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" "mov z21.s, #0\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" "mov z22.s, #0\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "whilelt p6.b, %[temp], %[leftovers]\n" "mov z23.s, #0\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" - "add a_ptr1, a_ptr1, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" + "whilelt p0.s, %[temp], %[width]\n" "mov z24.s, #0\n" - "add a_ptr2, a_ptr2, #0x10\n" + "whilelt p4.b, %[temp], %[width]\n" "mov z25.s, #0\n" - "add a_ptr3, a_ptr3, #0x10\n" - "zip2 z12.b, z10.b, z8.b\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "zip1 z10.b, z10.b, z8.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "incw %[temp], all, mul #1\n" "mov z26.s, #0\n" - "ld1b z13.b, p4/z, [%[b_ptr2]]\n" + "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "mov z27.s, #0\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "zip2 z9.b, z9.b, z10.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "mov z28.s, #0\n" + "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "mov z29.s, #0\n" + "whilelt p1.s, %[temp], %[width]\n" + "zip2 z11.b, z8.b, z9.b\n" + "incw %[temp], all, mul #1\n" + "zip1 z9.b, z8.b, z9.b\n" + "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "mov z30.s, #0\n" - "zip2 z15.b, z12.b, z13.b\n" - "zip1 z13.b, z12.b, z13.b\n" + "ptrue p7.b\n" "mov z31.s, #0\n" - "cbz %[loops], 2f\n" - "b 3f\n" - "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" - "ld1w z16.s, p0/z, [%[c_ptr0]]\n" - "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" - "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" - "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" - "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" - "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" - "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" - "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" - "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" - "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" - "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" - "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" - "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" - "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" - "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" - "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" + "whilelt p2.s, %[temp], %[width]\n" + "zip2 z12.b, z10.b, z8.b\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "zip1 z10.b, z10.b, z8.b\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "incw %[temp], all, mul #1\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" - "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z9.b, p4/z, [%[b_ptr2]]\n" + "zip1 z8.b, z9.b, z10.b\n" + "whilelt p3.s, %[temp], %[width]\n" + "zip2 z9.b, z9.b, z10.b\n" "add a_ptr1, a_ptr1, #0x10\n" - "ld1b z10.b, p4/z, [%[b_ptr1]]\n" + "zip1 z10.b, z11.b, z12.b\n" "add a_ptr2, a_ptr2, #0x10\n" - "zip2 z11.b, z8.b, z9.b\n" + "zip2 z11.b, z11.b, z12.b\n" "add a_ptr3, a_ptr3, #0x10\n" - "zip1 z9.b, z8.b, z9.b\n" - "ld1b z8.b, p4/z, [%[b_ptr3]]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z12.b, z10.b, z8.b\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" - "zip1 z10.b, z10.b, z8.b\n" - "ld1b z14.b, p4/z, [%[b_ptr1]]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" - "zip1 z8.b, z9.b, z10.b\n" - "zip2 z9.b, z9.b, z10.b\n" - "zip1 z10.b, z11.b, z12.b\n" - "zip2 z11.b, z11.b, z12.b\n" - "ld1b z12.b, p4/z, [%[b_ptr0]]\n" + "ld1b z14.b, p4/z, [%[b_ptr1]]\n" + "cbz %[loops], 1f\n" + "2:\n" "zip2 z15.b, z12.b, z13.b\n" + "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" "zip1 z13.b, z12.b, z13.b\n" - "cbz %[loops], 2f\n" - "3:\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" + "udot z16.s, z8.b, z0.b[0]\n" + "ld1rqb z5.b, p7/z, [a_ptr1]\n" "udot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" + "ld1rqb z6.b, p7/z, [a_ptr2]\n" "udot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p7/z, [a_ptr1]\n" + "ld1rqb z7.b, p7/z, [a_ptr3]\n" "udot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p7/z, [a_ptr2]\n" + "subs %[loops], %[loops], #0x1\n" "zip2 z8.b, z14.b, z12.b\n" - "ld1rqb z7.b, p7/z, [a_ptr3]\n" + "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "zip1 z14.b, z14.b, z12.b\n" - "subs %[loops], %[loops], #0x1\n" + "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z17.s, z9.b, z0.b[0]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" + "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "zip1 z12.b, z13.b, z14.b\n" - "add %[b_ptr2], %[b_ptr2], %[ldb]\n" + "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "zip2 z13.b, z13.b, z14.b\n" - "add %[b_ptr1], %[b_ptr1], %[ldb]\n" + "add %[a_ptr0], %[a_ptr0], #0x20\n" "zip1 z14.b, z15.b, z8.b\n" - "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "add a_ptr1, a_ptr1, #0x20\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" - "add %[a_ptr0], %[a_ptr0], #0x20\n" + "add a_ptr2, a_ptr2, #0x20\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z22.s, z10.b, z1.b[0]\n" - "add a_ptr1, a_ptr1, #0x20\n" + "add a_ptr3, a_ptr3, #0x20\n" "udot z26.s, z10.b, z2.b[0]\n" - "add a_ptr2, a_ptr2, #0x20\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" "udot z19.s, z11.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "udot z23.s, z11.b, z1.b[0]\n" - "add a_ptr3, a_ptr3, #0x20\n" "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" "zip2 z11.b, z8.b, z9.b\n" @@ -3649,29 +3465,29 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" + "b.ne 2b\n" + "1:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" - "b.ne 3b\n" - "2:\n" - "cbz %[regs], 4f\n" - "udot z16.s, z8.b, z0.b[0]\n" "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" + "cbz %[regs], 3f\n" + "udot z16.s, z8.b, z0.b[0]\n" "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p7/z, [a_ptr1]\n" - "udot z28.s, z8.b, z3.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p7/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "udot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p7/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" "zip1 z14.b, z15.b, z8.b\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3797,9 +3613,13 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z16.s, z8.b, z4.b[0]\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z28.s, z8.b, z7.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "zip2 z8.b, z14.b, z12.b\n" + "addvl a_ptr3, a_ptr3, #2\n" "zip1 z14.b, z14.b, z12.b\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -3893,7 +3713,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" - "cbz %[blocks], 5f\n" + "cbz %[blocks], 4f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3927,7 +3747,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" - "b.eq 6f\n" + "b.eq 5f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -3961,7 +3781,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" - "b.eq 7f\n" + "b.eq 6f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -3994,31 +3814,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 9f\n" + "b.eq 8f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 10f\n" + "b.eq 9f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "10:\n" + "b 10f\n" + "9:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 11f\n" - "9:\n" + "b 10f\n" + "8:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "11:\n" + "10:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4044,33 +3864,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" - "b 8f\n" - "7:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "6:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 12f\n" + "b.eq 11f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 13f\n" + "b.eq 12f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "13:\n" + "b 13f\n" + "12:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 14f\n" - "12:\n" + "b 13f\n" + "11:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "14:\n" + "13:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4096,33 +3916,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" - "b 8f\n" - "6:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "5:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 15f\n" + "b.eq 14f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 16f\n" + "b.eq 15f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "16:\n" + "b 16f\n" + "15:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 17f\n" - "15:\n" + "b 16f\n" + "14:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "17:\n" + "16:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4148,33 +3968,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" - "b 8f\n" - "5:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "4:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 18f\n" + "b.eq 17f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 19f\n" + "b.eq 18f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "19:\n" + "b 19f\n" + "18:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 20f\n" - "18:\n" + "b 19f\n" + "17:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "20:\n" + "19:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4200,37 +4020,40 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" - "b 8f\n" - "4:\n" + "b 7f\n" + "3:\n" "udot z16.s, z8.b, z0.b[0]\n" - "ld1b z12.b, p4/z, [%[b_ptr3]]\n" - "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z24.s, z8.b, z2.b[0]\n" + "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" - "udot z28.s, z8.b, z3.b[0]\n" + "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z6.b, p6/z, [a_ptr2]\n" - "zip2 z8.b, z14.b, z12.b\n" + "udot z28.s, z8.b, z3.b[0]\n" "ld1rqb z7.b, p6/z, [a_ptr3]\n" - "zip1 z14.b, z14.b, z12.b\n" + "zip2 z8.b, z14.b, z12.b\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "udot z17.s, z9.b, z0.b[0]\n" + "zip1 z14.b, z14.b, z12.b\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" - "zip1 z12.b, z13.b, z14.b\n" + "udot z17.s, z9.b, z0.b[0]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" - "zip2 z13.b, z13.b, z14.b\n" + "zip1 z12.b, z13.b, z14.b\n" "add %[b_ptr3], %[b_ptr3], %[ldb]\n" + "zip2 z13.b, z13.b, z14.b\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "zip1 z14.b, z15.b, z8.b\n" + "addvl a_ptr1, a_ptr1, #1\n" "zip2 z15.b, z15.b, z8.b\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "udot z25.s, z9.b, z2.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "udot z18.s, z10.b, z0.b[0]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" @@ -4309,7 +4132,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" - "cbz %[blocks], 21f\n" + "cbz %[blocks], 20f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4343,7 +4166,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" - "b.eq 22f\n" + "b.eq 21f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" @@ -4377,7 +4200,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" - "b.eq 23f\n" + "b.eq 22f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" @@ -4410,31 +4233,31 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" - "cbz %[odds], 8f\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 24f\n" + "b.eq 23f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 25f\n" + "b.eq 24f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "25:\n" + "b 25f\n" + "24:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 26f\n" - "24:\n" + "b 25f\n" + "23:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "26:\n" + "25:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4460,33 +4283,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" - "b 8f\n" - "23:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "22:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 27f\n" + "b.eq 26f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 28f\n" + "b.eq 27f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "28:\n" + "b 28f\n" + "27:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 29f\n" - "27:\n" + "b 28f\n" + "26:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "29:\n" + "28:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4512,33 +4335,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" - "b 8f\n" - "22:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "21:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 30f\n" + "b.eq 29f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 31f\n" + "b.eq 30f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z13.b, p4/z, [%[b_ptr2]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "31:\n" + "b 31f\n" + "30:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" "ld1b z14.b, p4/z, [%[b_ptr1]]\n" - "b 32f\n" - "30:\n" + "b 31f\n" + "29:\n" "mov z13.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z14.b, #0\n" "ld1b z12.b, p4/z, [%[b_ptr0]]\n" - "32:\n" + "31:\n" "zip2 z15.b, z12.b, z13.b\n" "zip1 z13.b, z12.b, z13.b\n" "mov z12.b, #0\n" @@ -4564,33 +4387,33 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" - "b 8f\n" - "21:\n" - "cbz %[odds], 8f\n" + "b 7f\n" + "20:\n" + "cbz %[odds], 7f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 33f\n" + "b.eq 32f\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 34f\n" + "b.eq 33f\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr2], %[b_ptr2], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z9.b, p4/z, [%[b_ptr2]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "34:\n" + "b 34f\n" + "33:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "add %[b_ptr1], %[b_ptr1], %[ldb]\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" "ld1b z10.b, p4/z, [%[b_ptr1]]\n" - "b 35f\n" - "33:\n" + "b 34f\n" + "32:\n" "mov z9.b, #0\n" "add %[b_ptr0], %[b_ptr0], %[ldb]\n" "mov z10.b, #0\n" "ld1b z8.b, p4/z, [%[b_ptr0]]\n" - "35:\n" + "34:\n" "zip2 z11.b, z8.b, z9.b\n" "zip1 z9.b, z8.b, z9.b\n" "mov z8.b, #0\n" @@ -4616,7 +4439,7 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" - "8:\n" + "7:\n" "st1w z16.s, p0, [%[c_ptr0]]\n" "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n" "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n" @@ -4641,11 +4464,12 @@ void sve_native_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, int ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [b_ptr1] "+r" (b_ptr1), [b_ptr2] "+r" (b_ptr2), [b_ptr3] "+r" (b_ptr3), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks), [odds] "+r" (odds) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers), [ldb] "r" (ldbb) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp deleted file mode 100644 index e2cc1d14e2..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_fp32_mla_1VLx4/generic.cpp +++ /dev/null @@ -1,4264 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __ARM_FEATURE_SVE - -#include <algorithm> - - -#include "../../asmlib.hpp" -#include "../../utils.hpp" - -namespace arm_gemm { - -void sve_smallK_fp32_mla_1VLx4(const float *A, int lda, const float *B, int ldb, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); - - const long loops_count = M / 4; - const long oddrow_count = M % 4; - const long ldab = lda * sizeof(float); - const long ldcb = ldc * sizeof(float); - const long odd_depth = K % 4; - const float *betaptr = β - long ldbb = ldb * sizeof(float); - - for (int x0=0; x0<N; x0+=(get_vector_length<float>() * 1)) { - const long width = std::min((unsigned long)N-x0, (get_vector_length<float>() * 1)); - long loops = loops_count; - long oddrows = oddrow_count; - long temp = 0; - const float *b_ptr0 = B + x0; - - const float *a_ptr0 = A; - - float *c_ptr0 = C + x0; - - switch(K) { - case 1: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 2: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 3: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 4: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 5: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 6: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 7: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 8: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 9: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 10: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 11: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 12: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 13: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 14: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 15: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 16: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 17: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 18: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 19: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 20: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 21: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 22: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z25.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 23: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z25.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z26.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - default: - case 24: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z5.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z6.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z7.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z8.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z9.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z10.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z11.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z12.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z13.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z14.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z15.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z16.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z17.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z18.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z19.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z20.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z21.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z22.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z23.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z24.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z25.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z26.s, p0/z, [%[b_ptr0]]\n" - "add %[b_ptr0], %[b_ptr0], %[ldb]\n" - "ld1w z27.s, p0/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "fmla z28.s, z27.s, z0.s[3]\n" - "fmla z29.s, z27.s, z1.s[3]\n" - "fmla z30.s, z27.s, z2.s[3]\n" - "fmla z31.s, z27.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "fmla z28.s, z27.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb), [ldb] "r" (ldbb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - } - } -} - -} // namespace arm_gemm - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp deleted file mode 100644 index 3e7e713106..0000000000 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx4/generic.cpp +++ /dev/null @@ -1,4004 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifdef __ARM_FEATURE_SVE - -#include <algorithm> - - -#include "../../asmlib.hpp" -#include "../../utils.hpp" - -namespace arm_gemm { - -void sve_smallK_hybrid_fp32_mla_1VLx4(const float *A, int lda, const float *B, float *C, int ldc, float beta, int M, int N, int K) { - const long beta0 = (beta == 0.0f); - - const long loops_count = M / 4; - const long oddrow_count = M % 4; - const long ldab = lda * sizeof(float); - const long ldcb = ldc * sizeof(float); - const int K_stride = K; - const long odd_depth = K % 4; - const float *betaptr = β - - for (int x0=0; x0<N; x0+=(get_vector_length<float>() * 1)) { - const long width = std::min((unsigned long)N-x0, (get_vector_length<float>() * 1)); - long loops = loops_count; - long oddrows = oddrow_count; - long temp = 0; - const float *b_ptr0 = B + (K_stride * x0); - - const float *a_ptr0 = A; - - float *c_ptr0 = C + x0; - - switch(K) { - case 1: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 2: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 3: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p6/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p6/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p6/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 4: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "subs %[loops], %[loops], #0x1\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 5: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 6: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 7: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 8: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 9: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 10: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 11: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p6/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 12: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 13: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 14: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 15: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p6/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 16: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 17: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 18: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 19: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 20: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 21: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 22: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - case 23: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p6/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p6/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p6/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - default: - case 24: - __asm __volatile ( - "a_ptr1 .req X0\n" - "a_ptr2 .req X1\n" - "a_ptr3 .req X2\n" - "c_ptr1 .req X3\n" - "c_ptr2 .req X4\n" - "c_ptr3 .req X5\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "whilelt p6.s, %[temp], %[odd_depth]\n" - "whilelt p0.s, %[temp], %[width]\n" - "ptrue p7.s\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" - "ld1w z4.s, p7/z, [%[b_ptr0]]\n" - "add a_ptr3, a_ptr2, %[lda]\n" - "ld1w z5.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add c_ptr3, c_ptr2, %[ldc]\n" - "ld1w z6.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z7.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z8.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z9.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z10.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z11.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" - "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n" - "ld1w z16.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n" - "ld1w z17.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1w z18.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n" - "ld1w z19.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "ld1w z20.s, p7/z, [%[b_ptr0]]\n" - "ld1w z21.s, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1w z22.s, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "ld1w z23.s, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "ld1w z24.s, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1w z25.s, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1w z26.s, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1w z27.s, p7/z, [%[b_ptr0], #7, MUL VL]\n" - "cbz %[loops], 1f\n" - "2:\n" - "cbz %[beta0], 3f\n" - "mov z28.s, #0\n" - "mov z29.s, #0\n" - "mov z30.s, #0\n" - "mov z31.s, #0\n" - "b 4f\n" - "3:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "ld1w z29.s, p0/z, [c_ptr1]\n" - "ld1w z30.s, p0/z, [c_ptr2]\n" - "ld1w z31.s, p0/z, [c_ptr3]\n" - "4:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[loops], %[loops], #0x1\n" - "ld1rqw z1.s, p7/z, [a_ptr1]\n" - "ld1rqw z2.s, p7/z, [a_ptr2]\n" - "ld1rqw z3.s, p7/z, [a_ptr3]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z29.s, z4.s, z1.s[0]\n" - "fmla z30.s, z4.s, z2.s[0]\n" - "fmla z31.s, z4.s, z3.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z29.s, z5.s, z1.s[1]\n" - "fmla z30.s, z5.s, z2.s[1]\n" - "fmla z31.s, z5.s, z3.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z29.s, z6.s, z1.s[2]\n" - "fmla z30.s, z6.s, z2.s[2]\n" - "fmla z31.s, z6.s, z3.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x10]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x10]\n" - "fmla z30.s, z7.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x10]\n" - "fmla z31.s, z7.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x10]\n" - "fmla z28.s, z8.s, z0.s[0]\n" - "fmla z29.s, z8.s, z1.s[0]\n" - "fmla z30.s, z8.s, z2.s[0]\n" - "fmla z31.s, z8.s, z3.s[0]\n" - "fmla z28.s, z9.s, z0.s[1]\n" - "fmla z29.s, z9.s, z1.s[1]\n" - "fmla z30.s, z9.s, z2.s[1]\n" - "fmla z31.s, z9.s, z3.s[1]\n" - "fmla z28.s, z10.s, z0.s[2]\n" - "fmla z29.s, z10.s, z1.s[2]\n" - "fmla z30.s, z10.s, z2.s[2]\n" - "fmla z31.s, z10.s, z3.s[2]\n" - "fmla z28.s, z11.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x20]\n" - "fmla z29.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x20]\n" - "fmla z30.s, z11.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x20]\n" - "fmla z31.s, z11.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x20]\n" - "fmla z28.s, z12.s, z0.s[0]\n" - "fmla z29.s, z12.s, z1.s[0]\n" - "fmla z30.s, z12.s, z2.s[0]\n" - "fmla z31.s, z12.s, z3.s[0]\n" - "fmla z28.s, z13.s, z0.s[1]\n" - "fmla z29.s, z13.s, z1.s[1]\n" - "fmla z30.s, z13.s, z2.s[1]\n" - "fmla z31.s, z13.s, z3.s[1]\n" - "fmla z28.s, z14.s, z0.s[2]\n" - "fmla z29.s, z14.s, z1.s[2]\n" - "fmla z30.s, z14.s, z2.s[2]\n" - "fmla z31.s, z14.s, z3.s[2]\n" - "fmla z28.s, z15.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z29.s, z15.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x30]\n" - "fmla z30.s, z15.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x30]\n" - "fmla z31.s, z15.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x30]\n" - "fmla z28.s, z16.s, z0.s[0]\n" - "fmla z29.s, z16.s, z1.s[0]\n" - "fmla z30.s, z16.s, z2.s[0]\n" - "fmla z31.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z0.s[1]\n" - "fmla z29.s, z17.s, z1.s[1]\n" - "fmla z30.s, z17.s, z2.s[1]\n" - "fmla z31.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z0.s[2]\n" - "fmla z29.s, z18.s, z1.s[2]\n" - "fmla z30.s, z18.s, z2.s[2]\n" - "fmla z31.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z29.s, z19.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x40]\n" - "fmla z30.s, z19.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x40]\n" - "fmla z31.s, z19.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x40]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z29.s, z20.s, z1.s[0]\n" - "fmla z30.s, z20.s, z2.s[0]\n" - "fmla z31.s, z20.s, z3.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z29.s, z21.s, z1.s[1]\n" - "fmla z30.s, z21.s, z2.s[1]\n" - "fmla z31.s, z21.s, z3.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z29.s, z22.s, z1.s[2]\n" - "fmla z30.s, z22.s, z2.s[2]\n" - "fmla z31.s, z22.s, z3.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda], LSL #2\n" - "fmla z29.s, z23.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [a_ptr1, #0x50]\n" - "fmla z30.s, z23.s, z2.s[3]\n" - "ld1rqw z2.s, p7/z, [a_ptr2, #0x50]\n" - "fmla z31.s, z23.s, z3.s[3]\n" - "ld1rqw z3.s, p7/z, [a_ptr3, #0x50]\n" - "fmla z28.s, z24.s, z0.s[0]\n" - "add a_ptr1, a_ptr1, %[lda], LSL #2\n" - "fmla z29.s, z24.s, z1.s[0]\n" - "add a_ptr2, a_ptr2, %[lda], LSL #2\n" - "fmla z30.s, z24.s, z2.s[0]\n" - "add a_ptr3, a_ptr3, %[lda], LSL #2\n" - "fmla z31.s, z24.s, z3.s[0]\n" - "fmla z28.s, z25.s, z0.s[1]\n" - "fmla z29.s, z25.s, z1.s[1]\n" - "fmla z30.s, z25.s, z2.s[1]\n" - "fmla z31.s, z25.s, z3.s[1]\n" - "fmla z28.s, z26.s, z0.s[2]\n" - "fmla z29.s, z26.s, z1.s[2]\n" - "fmla z30.s, z26.s, z2.s[2]\n" - "fmla z31.s, z26.s, z3.s[2]\n" - "fmla z28.s, z27.s, z0.s[3]\n" - "fmla z29.s, z27.s, z1.s[3]\n" - "fmla z30.s, z27.s, z2.s[3]\n" - "fmla z31.s, z27.s, z3.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc], LSL #2\n" - "st1w z29.s, p0, [c_ptr1]\n" - "add c_ptr1, c_ptr1, %[ldc], LSL #2\n" - "st1w z30.s, p0, [c_ptr2]\n" - "add c_ptr2, c_ptr2, %[ldc], LSL #2\n" - "st1w z31.s, p0, [c_ptr3]\n" - "add c_ptr3, c_ptr3, %[ldc], LSL #2\n" - "b.ne 2b\n" - "1:\n" - "cbz %[oddrows], 5f\n" - "6:\n" - "cbz %[beta0], 7f\n" - "mov z28.s, #0\n" - "b 8f\n" - "7:\n" - "ld1w z28.s, p0/z, [%[c_ptr0]]\n" - "8:\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n" - "subs %[oddrows], %[oddrows], #0x1\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x10]\n" - "ld1rqw z2.s, p7/z, [%[a_ptr0], #0x20]\n" - "ld1rqw z3.s, p7/z, [%[a_ptr0], #0x30]\n" - "fmla z28.s, z4.s, z0.s[0]\n" - "fmla z28.s, z5.s, z0.s[1]\n" - "fmla z28.s, z6.s, z0.s[2]\n" - "fmla z28.s, z7.s, z0.s[3]\n" - "ld1rqw z0.s, p7/z, [%[a_ptr0], #0x40]\n" - "fmla z28.s, z8.s, z1.s[0]\n" - "fmla z28.s, z9.s, z1.s[1]\n" - "fmla z28.s, z10.s, z1.s[2]\n" - "fmla z28.s, z11.s, z1.s[3]\n" - "ld1rqw z1.s, p7/z, [%[a_ptr0], #0x50]\n" - "add %[a_ptr0], %[a_ptr0], %[lda]\n" - "fmla z28.s, z12.s, z2.s[0]\n" - "fmla z28.s, z13.s, z2.s[1]\n" - "fmla z28.s, z14.s, z2.s[2]\n" - "fmla z28.s, z15.s, z2.s[3]\n" - "fmla z28.s, z16.s, z3.s[0]\n" - "fmla z28.s, z17.s, z3.s[1]\n" - "fmla z28.s, z18.s, z3.s[2]\n" - "fmla z28.s, z19.s, z3.s[3]\n" - "fmla z28.s, z20.s, z0.s[0]\n" - "fmla z28.s, z21.s, z0.s[1]\n" - "fmla z28.s, z22.s, z0.s[2]\n" - "fmla z28.s, z23.s, z0.s[3]\n" - "fmla z28.s, z24.s, z1.s[0]\n" - "fmla z28.s, z25.s, z1.s[1]\n" - "fmla z28.s, z26.s, z1.s[2]\n" - "fmla z28.s, z27.s, z1.s[3]\n" - "st1w z28.s, p0, [%[c_ptr0]]\n" - "add %[c_ptr0], %[c_ptr0], %[ldc]\n" - "b.ne 6b\n" - "5:\n" - ".unreq a_ptr1\n" - ".unreq a_ptr2\n" - ".unreq a_ptr3\n" - ".unreq c_ptr1\n" - ".unreq c_ptr2\n" - ".unreq c_ptr3\n" - : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [temp] "+r" (temp), [oddrows] "+r" (oddrows) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [odd_depth] "r" (odd_depth), [lda] "r" (ldab), [ldc] "r" (ldcb) - : "x0", "x1", "x2", "x3", "x4", "x5", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" - ); - break; - } - } -} - -} // namespace arm_gemm - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp index ca90ee18ce..fc18cbdbbf 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp @@ -31,7 +31,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); +void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); class smallK_hybrid_s8s32_dot_1VLx8 { @@ -39,7 +39,7 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int32_t, int, int, int); + typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -57,15 +57,27 @@ public: return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {}; // Default to the generic kernel kern_type kernel=sve_smallK_hybrid_s8s32_dot_1VLx8; - smallK_hybrid_s8s32_dot_1VLx8(const CPUInfo *ci) - { - - } + smallK_hybrid_s8s32_dot_1VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp index 04605b2ac1..d9813a5cdf 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp @@ -25,13 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int32_t beta, int M, int N, int K) { +void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)get_vector_length<int32_t>()) - 1; const long ldab = lda * sizeof(int8_t); const long ldcb = ldc * sizeof(int32_t); diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp index 453301417d..51d3e736ed 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp @@ -31,7 +31,7 @@ namespace arm_gemm { // Actual kernel implementations -void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); +void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); class smallK_hybrid_u8u32_dot_1VLx8 { @@ -39,7 +39,7 @@ public: typedef uint8_t operand_type; typedef uint32_t result_type; - typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, uint32_t, int, int, int); + typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool); /* Kernel blocking parameters */ static constexpr unsigned int out_height() @@ -57,15 +57,27 @@ public: return 4; } + static constexpr bool supports_append() + { + return false; + } + + static constexpr bool supports_bias() + { + return false; + } + + static constexpr bool supports_activation() + { + return false; + } + StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {}; // Default to the generic kernel kern_type kernel=sve_smallK_hybrid_u8u32_dot_1VLx8; - smallK_hybrid_u8u32_dot_1VLx8(const CPUInfo *ci) - { - - } + smallK_hybrid_u8u32_dot_1VLx8(const CPUInfo *ci) { UNUSED(ci); } }; } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp index 7c965d38a6..6eed457152 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp @@ -25,13 +25,15 @@ #include <algorithm> +#include "arm_gemm.hpp" + #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { +void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) { const long loops_count = iceildiv(N, (int)get_vector_length<uint32_t>()) - 1; const long ldab = lda * sizeof(uint8_t); const long ldcb = ldc * sizeof(uint32_t); |