diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp')
| -rw-r--r-- | src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp | 454 |
1 files changed, 215 insertions, 239 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp index d34d0e5fc7..bf3e8ca26a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,14 +25,14 @@ #include <algorithm> +#include "arm_gemm.hpp" #include <cstdint> #include "../../asmlib.hpp" #include "../../utils.hpp" namespace arm_gemm { -void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, uint32_t beta, int M, int N, int K) { - const long beta0 = (beta == 0u); +void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool append) { const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -46,17 +46,16 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin const unsigned long ldab = lda * sizeof(uint8_t); uint32_t *c_ptr0 = C + (y * ldc); - const unsigned long ldcb = ldc * sizeof(uint32_t); for (int x0=0; x0<N; x0+=(4 * get_vector_length<uint32_t>())) { const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<uint32_t>())); - const uint32_t *betaptr = β long loops = loops_count; long regs = regs_count; long temp = 0; long blocks = blocks_count; const uint8_t *a_ptr0 = a_ptr0_base; const uint8_t *b_ptr0 = B + (K_stride * x0); + const unsigned long ldcb = ldc * sizeof(uint32_t); switch(M-y) { case 1: @@ -70,7 +69,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -84,30 +83,23 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -224,13 +216,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" - "udot z17.s, z13.b, z4.b[1]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z18.s, z14.b, z4.b[1]\n" + "udot z17.s, z13.b, z4.b[1]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "udot z19.s, z15.b, z4.b[1]\n" + "udot z18.s, z14.b, z4.b[1]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "udot z19.s, z15.b, z4.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" @@ -241,45 +232,48 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" + "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n" "udot z17.s, z13.b, z4.b[3]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z18.s, z14.b, z4.b[3]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z19.s, z15.b, z4.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z0.b[0]\n" "udot z18.s, z10.b, z0.b[0]\n" "udot z19.s, z11.b, z0.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z17.s, z13.b, z0.b[1]\n" "udot z18.s, z14.b, z0.b[1]\n" "udot z19.s, z15.b, z0.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z18.s, z10.b, z0.b[2]\n" "udot z19.s, z11.b, z0.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" @@ -289,65 +283,67 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z19.s, z11.b, z0.b[0]\n" + "udot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "udot z16.s, z12.b, z0.b[1]\n" + "udot z19.s, z11.b, z0.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "udot z17.s, z13.b, z0.b[1]\n" + "udot z16.s, z12.b, z0.b[1]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "udot z18.s, z14.b, z0.b[1]\n" + "udot z17.s, z13.b, z0.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "udot z19.s, z15.b, z0.b[1]\n" + "udot z18.s, z14.b, z0.b[1]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "udot z16.s, z8.b, z0.b[2]\n" + "udot z19.s, z15.b, z0.b[1]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "udot z17.s, z9.b, z0.b[2]\n" + "udot z16.s, z8.b, z0.b[2]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "udot z17.s, z9.b, z0.b[2]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z18.s, z10.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z19.s, z11.b, z0.b[2]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" "udot z18.s, z14.b, z0.b[3]\n" "udot z19.s, z15.b, z0.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z18.s, z10.b, z4.b[0]\n" "udot z19.s, z11.b, z4.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z17.s, z13.b, z4.b[1]\n" "udot z18.s, z14.b, z4.b[1]\n" "udot z19.s, z15.b, z4.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z18.s, z10.b, z4.b[2]\n" "udot z19.s, z11.b, z4.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z17.s, z13.b, z4.b[3]\n" "udot z18.s, z14.b, z4.b[3]\n" @@ -359,7 +355,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n" "addvl %[c_ptr0], %[c_ptr0], #4\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory" ); break; @@ -378,7 +374,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -397,41 +393,30 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "add a_ptr1, a_ptr1, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -601,9 +586,11 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z20.s, z8.b, z5.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z21.s, z9.b, z5.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z22.s, z10.b, z5.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z4.b[0]\n" @@ -622,6 +609,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z21.s, z9.b, z5.b[2]\n" @@ -638,13 +626,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z4.b[3]\n" "udot z23.s, z15.b, z5.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z17.s, z9.b, z0.b[0]\n" "udot z21.s, z9.b, z1.b[0]\n" @@ -653,12 +640,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z0.b[0]\n" "udot z23.s, z11.b, z1.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z17.s, z13.b, z0.b[1]\n" "udot z21.s, z13.b, z1.b[1]\n" @@ -667,12 +654,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z0.b[1]\n" "udot z23.s, z15.b, z1.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z21.s, z9.b, z1.b[2]\n" @@ -681,10 +669,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z0.b[2]\n" "udot z23.s, z11.b, z1.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z17.s, z13.b, z0.b[3]\n" @@ -698,19 +686,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z0.b[0]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" - "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "udot z18.s, z10.b, z0.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "udot z18.s, z10.b, z0.b[0]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z22.s, z10.b, z1.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z23.s, z11.b, z1.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z20.s, z12.b, z1.b[1]\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "udot z17.s, z13.b, z0.b[1]\n" @@ -723,6 +713,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" "udot z21.s, z9.b, z1.b[2]\n" @@ -739,13 +730,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z0.b[3]\n" "udot z23.s, z15.b, z1.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z17.s, z9.b, z4.b[0]\n" "udot z21.s, z9.b, z5.b[0]\n" @@ -754,12 +744,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[0]\n" "udot z23.s, z11.b, z5.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z17.s, z13.b, z4.b[1]\n" "udot z21.s, z13.b, z5.b[1]\n" @@ -768,12 +758,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z15.b, z4.b[1]\n" "udot z23.s, z15.b, z5.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" "udot z21.s, z9.b, z5.b[2]\n" @@ -782,10 +773,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z19.s, z11.b, z4.b[2]\n" "udot z23.s, z11.b, z5.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z17.s, z13.b, z4.b[3]\n" @@ -807,7 +798,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq a_ptr1\n" ".unreq c_ptr1\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory" ); break; @@ -830,7 +821,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -852,54 +843,39 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "mov z25.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z26.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z27.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "mov z27.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" "add a_ptr2, a_ptr2, #0x10\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1120,10 +1096,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z4.b[0]\n" "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "udot z25.s, z9.b, z6.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z4.b[0]\n" @@ -1151,6 +1130,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" @@ -1175,13 +1155,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[3]\n" "udot z27.s, z15.b, z6.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z24.s, z8.b, z2.b[0]\n" "udot z17.s, z9.b, z0.b[0]\n" @@ -1194,12 +1173,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z24.s, z12.b, z2.b[1]\n" "udot z17.s, z13.b, z0.b[1]\n" @@ -1212,12 +1191,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[1]\n" "udot z27.s, z15.b, z2.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" @@ -1230,10 +1210,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z1.b[2]\n" "udot z27.s, z11.b, z2.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z24.s, z12.b, z2.b[3]\n" @@ -1253,18 +1233,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z20.s, z8.b, z1.b[0]\n" "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n" "udot z24.s, z8.b, z2.b[0]\n" - "ld1rqb z5.b, p6/z, [a_ptr1]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "udot z25.s, z9.b, z2.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z0.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z26.s, z10.b, z2.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z23.s, z11.b, z1.b[0]\n" "udot z27.s, z11.b, z2.b[0]\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" @@ -1285,6 +1268,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z17.s, z9.b, z0.b[2]\n" @@ -1309,13 +1293,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z1.b[3]\n" "udot z27.s, z15.b, z2.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z24.s, z8.b, z6.b[0]\n" "udot z17.s, z9.b, z4.b[0]\n" @@ -1328,12 +1311,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z5.b[0]\n" "udot z27.s, z11.b, z6.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z24.s, z12.b, z6.b[1]\n" "udot z17.s, z13.b, z4.b[1]\n" @@ -1346,12 +1329,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z15.b, z5.b[1]\n" "udot z27.s, z15.b, z6.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z17.s, z9.b, z4.b[2]\n" @@ -1364,10 +1348,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z23.s, z11.b, z5.b[2]\n" "udot z27.s, z11.b, z6.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z24.s, z12.b, z6.b[3]\n" @@ -1399,7 +1383,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq c_ptr1\n" ".unreq c_ptr2\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory" ); break; @@ -1427,7 +1411,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "whilelt p2.s, %[temp], %[width]\n" "incw %[temp], all, mul #1\n" "whilelt p3.s, %[temp], %[width]\n" - "cbz %[beta0], 1f\n" + "cbnz %[append], 1f\n" "mov z16.s, #0\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" "mov z17.s, #0\n" @@ -1451,68 +1435,49 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "mov z26.s, #0\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "mov z27.s, #0\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" - "mov z28.s, #0\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "mov z29.s, #0\n" + "mov z28.s, #0\n" "add a_ptr1, a_ptr1, #0x10\n" - "mov z30.s, #0\n" + "mov z29.s, #0\n" "add a_ptr2, a_ptr2, #0x10\n" - "mov z31.s, #0\n" + "mov z30.s, #0\n" "add a_ptr3, a_ptr3, #0x10\n" + "mov z31.s, #0\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "b 3f\n" "1:\n" - "ld1rw z15.s, p7/z, [%[betaptr]]\n" "ld1w z16.s, p0/z, [%[c_ptr0]]\n" "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n" "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n" "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n" - "mul z16.s, p7/m, z16.s, z15.s\n" "ld1w z20.s, p0/z, [c_ptr1]\n" - "mul z17.s, p7/m, z17.s, z15.s\n" "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n" - "mul z18.s, p7/m, z18.s, z15.s\n" "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n" - "mul z19.s, p7/m, z19.s, z15.s\n" "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n" - "mul z20.s, p7/m, z20.s, z15.s\n" "ld1w z24.s, p0/z, [c_ptr2]\n" - "mul z21.s, p7/m, z21.s, z15.s\n" "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n" - "mul z22.s, p7/m, z22.s, z15.s\n" "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n" - "mul z23.s, p7/m, z23.s, z15.s\n" "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n" - "mul z24.s, p7/m, z24.s, z15.s\n" "ld1w z28.s, p0/z, [c_ptr3]\n" - "mul z25.s, p7/m, z25.s, z15.s\n" "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n" - "mul z26.s, p7/m, z26.s, z15.s\n" "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n" - "mul z27.s, p7/m, z27.s, z15.s\n" "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n" - "mul z28.s, p7/m, z28.s, z15.s\n" "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n" - "mul z29.s, p7/m, z29.s, z15.s\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1rqb z1.b, p7/z, [a_ptr1]\n" - "mul z30.s, p7/m, z30.s, z15.s\n" + "add a_ptr1, a_ptr1, #0x10\n" "ld1rqb z2.b, p7/z, [a_ptr2]\n" - "mul z31.s, p7/m, z31.s, z15.s\n" + "add a_ptr2, a_ptr2, #0x10\n" "ld1rqb z3.b, p7/z, [a_ptr3]\n" + "add a_ptr3, a_ptr3, #0x10\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" - "add %[a_ptr0], %[a_ptr0], #0x10\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "add a_ptr1, a_ptr1, #0x10\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" - "add a_ptr2, a_ptr2, #0x10\n" "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" - "add a_ptr3, a_ptr3, #0x10\n" "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "addvl %[b_ptr0], %[b_ptr0], #8\n" "cbz %[loops], 2f\n" "3:\n" @@ -1786,11 +1751,15 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z16.s, z8.b, z4.b[0]\n" "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n" "udot z20.s, z8.b, z5.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #2\n" "udot z24.s, z8.b, z6.b[0]\n" + "addvl a_ptr1, a_ptr1, #2\n" "udot z28.s, z8.b, z7.b[0]\n" "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z4.b[0]\n" + "addvl a_ptr2, a_ptr2, #2\n" "udot z21.s, z9.b, z5.b[0]\n" + "addvl a_ptr3, a_ptr3, #2\n" "udot z25.s, z9.b, z6.b[0]\n" "udot z29.s, z9.b, z7.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" @@ -1825,6 +1794,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z31.s, z15.b, z7.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z28.s, z8.b, z7.b[2]\n" @@ -1857,13 +1827,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[3]\n" "udot z31.s, z15.b, z7.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z0.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[0]\n" "udot z24.s, z8.b, z2.b[0]\n" "udot z28.s, z8.b, z3.b[0]\n" @@ -1880,12 +1849,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z2.b[0]\n" "udot z31.s, z11.b, z3.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z0.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z1.b[1]\n" "udot z24.s, z12.b, z2.b[1]\n" "udot z28.s, z12.b, z3.b[1]\n" @@ -1902,12 +1871,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[1]\n" "udot z31.s, z15.b, z3.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z28.s, z8.b, z3.b[2]\n" @@ -1924,10 +1894,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z2.b[2]\n" "udot z31.s, z11.b, z3.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z0.b[3]\n" "udot z20.s, z12.b, z1.b[3]\n" "udot z24.s, z12.b, z2.b[3]\n" @@ -1953,17 +1923,21 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z24.s, z8.b, z2.b[0]\n" "ld1rqb z5.b, p6/z, [a_ptr1]\n" "udot z28.s, z8.b, z3.b[0]\n" - "ld1rqb z6.b, p6/z, [a_ptr2]\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "udot z17.s, z9.b, z0.b[0]\n" - "ld1rqb z7.b, p6/z, [a_ptr3]\n" + "ld1rqb z6.b, p6/z, [a_ptr2]\n" "udot z21.s, z9.b, z1.b[0]\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "ld1rqb z7.b, p6/z, [a_ptr3]\n" "udot z25.s, z9.b, z2.b[0]\n" + "addvl %[a_ptr0], %[a_ptr0], #1\n" "udot z29.s, z9.b, z3.b[0]\n" "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" "udot z18.s, z10.b, z0.b[0]\n" + "addvl a_ptr1, a_ptr1, #1\n" "udot z22.s, z10.b, z1.b[0]\n" + "addvl a_ptr2, a_ptr2, #1\n" "udot z26.s, z10.b, z2.b[0]\n" + "addvl a_ptr3, a_ptr3, #1\n" "udot z30.s, z10.b, z3.b[0]\n" "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z19.s, z11.b, z0.b[0]\n" @@ -1992,6 +1966,7 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z31.s, z15.b, z3.b[1]\n" "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z16.s, z8.b, z0.b[2]\n" + "addvl %[b_ptr0], %[b_ptr0], #8\n" "udot z20.s, z8.b, z1.b[2]\n" "udot z24.s, z8.b, z2.b[2]\n" "udot z28.s, z8.b, z3.b[2]\n" @@ -2024,13 +1999,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z2.b[3]\n" "udot z31.s, z15.b, z3.b[3]\n" "cbz %[blocks], 5f\n" - "addvl %[b_ptr0], %[b_ptr0], #16\n" + "ld1b z8.b, p7/z, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" "udot z16.s, z8.b, z4.b[0]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[0]\n" "udot z24.s, z8.b, z6.b[0]\n" "udot z28.s, z8.b, z7.b[0]\n" @@ -2047,12 +2021,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z6.b[0]\n" "udot z31.s, z11.b, z7.b[0]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" "udot z16.s, z12.b, z4.b[1]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" "udot z20.s, z12.b, z5.b[1]\n" "udot z24.s, z12.b, z6.b[1]\n" "udot z28.s, z12.b, z7.b[1]\n" @@ -2069,12 +2043,13 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z15.b, z6.b[1]\n" "udot z31.s, z15.b, z7.b[1]\n" "b.eq 5f\n" - "ld1b z8.b, p7/z, [%[b_ptr0]]\n" + "addvl %[b_ptr0], %[b_ptr0], #16\n" "subs %[blocks], %[blocks], #0x1\n" - "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n" - "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n" + "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n" + "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n" + "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n" + "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n" "udot z16.s, z8.b, z4.b[2]\n" - "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n" "udot z20.s, z8.b, z5.b[2]\n" "udot z24.s, z8.b, z6.b[2]\n" "udot z28.s, z8.b, z7.b[2]\n" @@ -2091,10 +2066,10 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin "udot z27.s, z11.b, z6.b[2]\n" "udot z31.s, z11.b, z7.b[2]\n" "b.eq 5f\n" - "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n" - "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n" - "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n" - "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n" + "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n" + "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n" + "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n" + "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n" "udot z16.s, z12.b, z4.b[3]\n" "udot z20.s, z12.b, z5.b[3]\n" "udot z24.s, z12.b, z6.b[3]\n" @@ -2136,11 +2111,12 @@ void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uin ".unreq c_ptr2\n" ".unreq c_ptr3\n" : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks) - : [betaptr] "r" (betaptr), [width] "r" (width), [beta0] "r" (beta0), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) + : [width] "r" (width), [append] "r" (static_cast<uint64_t>(append)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers) : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory" ); break; } + } } } |