diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp')
| -rw-r--r-- | src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp | 375 |
1 files changed, 244 insertions, 131 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp index 3ecf0151aa..7c08aa2165 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp @@ -35,7 +35,6 @@ namespace arm_gemm { void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool append) { UNUSED(bias); UNUSED(act); - const int K_stride = ((K + 3) / 4) * 4; const long loops_count = ((K + 16) / 32) - 1; K -= loops_count * 32; @@ -80,6 +79,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X1\n" "temploadreg2 .req X2\n" "temploadreg3 .req X3\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" @@ -95,8 +95,26 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ldr d14, [%[b_ptr0], #0x60]\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 1f\n" - "2:\n" + "cbz %[loops], 2f\n" + "b 3f\n" + "1:\n" + "ldr q16, [%[c_ptr0]]\n" + "ldr q17, [%[c_ptr0], #0x10]\n" + "ldr q18, [%[c_ptr0], #0x20]\n" + "ldr q19, [%[c_ptr0], #0x30]\n" + "ldr q0, [%[a_ptr0]]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr q8, [%[b_ptr0]]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "3:\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ins v14.d[1], temploadreg2\n" ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" @@ -236,14 +254,14 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" - "b.ne 2b\n" - "1:\n" + "b.ne 3b\n" + "2:\n" "ins v14.d[1], temploadreg2\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 3f\n" + "cbz %[regs], 4f\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" @@ -354,8 +372,8 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n" ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n" ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" - "b 4f\n" - "3:\n" + "b 5f\n" + "4:\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d8, [%[b_ptr0]]\n" ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" @@ -397,9 +415,9 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n" ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n" ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" - "4:\n" - "cbz %[blocks], 5f\n" - "6:\n" + "5:\n" + "cbz %[blocks], 6f\n" + "7:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -412,17 +430,17 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "b.ne 6b\n" - "5:\n" - "cbz %[odds], 7f\n" + "b.ne 7b\n" + "6:\n" + "cbz %[odds], 8f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 8f\n" + "b.eq 9f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 8f\n" + "b.eq 9f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" - "8:\n" + "9:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" @@ -431,7 +449,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "7:\n" + "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -454,74 +472,99 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X3\n" "temploadreg2 .req X4\n" "temploadreg3 .req X5\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q1, [a_ptr1]\n" "movi v18.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v19.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v20.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v21.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v22.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v23.4s, #0\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "ldr d14, [%[b_ptr0], #0x60]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "ldr temploadreg2, [%[b_ptr0], #0x68]\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "ldr q1, [a_ptr1]\n" + "add a_ptr1, a_ptr1, #0x10\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "b 3f\n" + "1:\n" + "ldr q16, [%[c_ptr0]]\n" + "ldr q17, [%[c_ptr0], #0x10]\n" + "ldr q18, [%[c_ptr0], #0x20]\n" + "ldr q19, [%[c_ptr0], #0x30]\n" + "ldr q20, [c_ptr1]\n" + "ldr q21, [c_ptr1, #0x10]\n" + "ldr q22, [c_ptr1, #0x20]\n" + "ldr q23, [c_ptr1, #0x30]\n" + "ldr q0, [%[a_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "ins v14.d[1], temploadreg2\n" + "ldr q1, [a_ptr1]\n" "add a_ptr1, a_ptr1, #0x10\n" + "ldr q8, [%[b_ptr0]]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 1f\n" - "2:\n" + "cbz %[loops], 2f\n" + "3:\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" - "ldr d15, [%[b_ptr0], #-0x10]\n" + "ins v14.d[1], temploadreg2\n" ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" - "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "ldr d15, [%[b_ptr0], #-0x10]\n" ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n" - "ldr d4, [%[a_ptr0]]\n" + "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n" - "ldr temploadreg0, [%[a_ptr0], #0x8]\n" + "ldr d4, [%[a_ptr0]]\n" ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n" - "ldr d5, [a_ptr1]\n" + "ldr temploadreg0, [%[a_ptr0], #0x8]\n" ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" - "ldr temploadreg1, [a_ptr1, #0x8]\n" + "ldr d5, [a_ptr1]\n" ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" - "ldr d8, [%[b_ptr0]]\n" + "ldr temploadreg1, [a_ptr1, #0x8]\n" ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "ins v4.d[1], temploadreg0\n" + "ldr d8, [%[b_ptr0]]\n" ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n" - "ldr temploadreg0, [%[b_ptr0], #0x8]\n" + "ins v4.d[1], temploadreg0\n" ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n" - "ldr d9, [%[b_ptr0], #0x10]\n" + "ldr temploadreg0, [%[b_ptr0], #0x8]\n" ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n" - "ins v5.d[1], temploadreg1\n" + "ldr d9, [%[b_ptr0], #0x10]\n" ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n" - "ldr temploadreg1, [%[b_ptr0], #0x18]\n" + "ins v5.d[1], temploadreg1\n" ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n" - "ldr d10, [%[b_ptr0], #0x20]\n" + "ldr temploadreg1, [%[b_ptr0], #0x18]\n" ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n" + "ldr d10, [%[b_ptr0], #0x20]\n" "ldr temploadreg2, [%[b_ptr0], #0x28]\n" - "ldr d11, [%[b_ptr0], #0x30]\n" "subs %[loops], %[loops], #0x1\n" - "ins v15.d[1], temploadreg3\n" + "ldr d11, [%[b_ptr0], #0x30]\n" "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n" - "ldr temploadreg3, [%[b_ptr0], #0x38]\n" + "ins v15.d[1], temploadreg3\n" "add %[a_ptr0], %[a_ptr0], #0x20\n" + "ldr temploadreg3, [%[b_ptr0], #0x38]\n" + "add a_ptr1, a_ptr1, #0x20\n" ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n" "ldr d12, [%[b_ptr0], #0x40]\n" ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n" "ins v8.d[1], temploadreg0\n" "ldr temploadreg0, [%[b_ptr0], #0x48]\n" - "add a_ptr1, a_ptr1, #0x20\n" - "ldr d13, [%[b_ptr0], #0x50]\n" "prfm PLDL1KEEP, [a_ptr1, #0x40]\n" + "ldr d13, [%[b_ptr0], #0x50]\n" ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n" "ins v9.d[1], temploadreg1\n" ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n" @@ -658,15 +701,15 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v11.d[1], temploadreg3\n" "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" + "b.ne 3b\n" + "2:\n" "ins v14.d[1], temploadreg2\n" - "b.ne 2b\n" - "1:\n" - "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" - "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" + "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [c_ptr1]\n" + "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 3f\n" + "cbz %[regs], 4f\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" @@ -813,8 +856,8 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n" ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" - "b 4f\n" - "3:\n" + "b 5f\n" + "4:\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" @@ -872,9 +915,9 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n" ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" - "4:\n" - "cbz %[blocks], 5f\n" - "6:\n" + "5:\n" + "cbz %[blocks], 6f\n" + "7:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -893,20 +936,20 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "b.ne 6b\n" - "5:\n" - "cbz %[odds], 7f\n" + "b.ne 7b\n" + "6:\n" + "cbz %[odds], 8f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 8f\n" + "b.eq 9f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 8f\n" + "b.eq 9f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" - "8:\n" + "9:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" @@ -919,7 +962,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n" ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" - "7:\n" + "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -950,40 +993,72 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X5\n" "temploadreg2 .req X6\n" "temploadreg3 .req X7\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q1, [a_ptr1]\n" "movi v18.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q2, [a_ptr2]\n" "movi v19.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v20.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v21.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v22.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v23.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v24.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v25.4s, #0\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "movi v26.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "movi v27.4s, #0\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" + "add a_ptr1, a_ptr1, #0x10\n" "ins v14.d[1], temploadreg2\n" - "add a_ptr2, a_ptr1, %[lda]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" - "ldr q2, [a_ptr2]\n" - "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "b 3f\n" + "1:\n" + "ldr q16, [%[c_ptr0]]\n" + "ldr q17, [%[c_ptr0], #0x10]\n" + "ldr q18, [%[c_ptr0], #0x20]\n" + "ldr q19, [%[c_ptr0], #0x30]\n" + "ldr q20, [c_ptr1]\n" + "ldr q21, [c_ptr1, #0x10]\n" + "ldr q22, [c_ptr1, #0x20]\n" + "ldr q23, [c_ptr1, #0x30]\n" + "ldr q24, [c_ptr2]\n" + "ldr q25, [c_ptr2, #0x10]\n" + "ldr q26, [c_ptr2, #0x20]\n" + "ldr q27, [c_ptr2, #0x30]\n" + "ldr q0, [%[a_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" + "ldr q1, [a_ptr1]\n" "add a_ptr1, a_ptr1, #0x10\n" + "ldr q2, [a_ptr2]\n" "add a_ptr2, a_ptr2, #0x10\n" + "ldr q8, [%[b_ptr0]]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 1f\n" - "2:\n" + "ins v14.d[1], temploadreg2\n" + "cbz %[loops], 2f\n" + "3:\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" @@ -1203,15 +1278,15 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v12.d[1], temploadreg0\n" "ins v13.d[1], temploadreg1\n" "ins v14.d[1], temploadreg2\n" - "b.ne 2b\n" - "1:\n" + "b.ne 3b\n" + "2:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" "prfm PSTL1KEEP, [c_ptr1]\n" "prfm PSTL1KEEP, [c_ptr2]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 3f\n" + "cbz %[regs], 4f\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" @@ -1394,8 +1469,8 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n" ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" - "b 4f\n" - "3:\n" + "b 5f\n" + "4:\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" @@ -1469,9 +1544,9 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n" ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" - "4:\n" - "cbz %[blocks], 5f\n" - "6:\n" + "5:\n" + "cbz %[blocks], 6f\n" + "7:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -1496,23 +1571,23 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "b.ne 6b\n" - "5:\n" - "cbz %[odds], 7f\n" + "b.ne 7b\n" + "6:\n" + "cbz %[odds], 8f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 8f\n" + "b.eq 9f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 8f\n" + "b.eq 9f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" - "8:\n" + "9:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" @@ -1529,7 +1604,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n" ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" - "7:\n" + "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" @@ -1569,48 +1644,86 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "temploadreg1 .req X7\n" "temploadreg2 .req X8\n" "temploadreg3 .req X9\n" + "add a_ptr1, %[a_ptr0], %[lda]\n" + "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr1, %[lda]\n" + "add c_ptr2, c_ptr1, %[ldc]\n" + "add a_ptr3, a_ptr2, %[lda]\n" + "add c_ptr3, c_ptr2, %[ldc]\n" + "cbnz %[append], 1f\n" "movi v16.4s, #0\n" "ldr q0, [%[a_ptr0]]\n" "movi v17.4s, #0\n" - "ldr q8, [%[b_ptr0]]\n" + "ldr q1, [a_ptr1]\n" "movi v18.4s, #0\n" - "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q2, [a_ptr2]\n" "movi v19.4s, #0\n" - "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q3, [a_ptr3]\n" "movi v20.4s, #0\n" - "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q8, [%[b_ptr0]]\n" "movi v21.4s, #0\n" - "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" "movi v22.4s, #0\n" - "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" "movi v23.4s, #0\n" - "ldr d14, [%[b_ptr0], #0x60]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" "movi v24.4s, #0\n" - "ldr temploadreg2, [%[b_ptr0], #0x68]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" "movi v25.4s, #0\n" - "add a_ptr1, %[a_ptr0], %[lda]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" "movi v26.4s, #0\n" - "ldr q1, [a_ptr1]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" "movi v27.4s, #0\n" - "ins v14.d[1], temploadreg2\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "movi v28.4s, #0\n" - "add a_ptr2, a_ptr1, %[lda]\n" + "add %[a_ptr0], %[a_ptr0], #0x10\n" "movi v29.4s, #0\n" - "ldr q2, [a_ptr2]\n" + "ins v14.d[1], temploadreg2\n" "movi v30.4s, #0\n" - "add a_ptr3, a_ptr2, %[lda]\n" + "add a_ptr1, a_ptr1, #0x10\n" "movi v31.4s, #0\n" - "ldr q3, [a_ptr3]\n" - "add c_ptr1, %[c_ptr0], %[ldc]\n" + "add a_ptr2, a_ptr2, #0x10\n" + "add a_ptr3, a_ptr3, #0x10\n" + "add %[b_ptr0], %[b_ptr0], #0x80\n" + "cbz %[loops], 2f\n" + "b 3f\n" + "1:\n" + "ldr q16, [%[c_ptr0]]\n" + "ldr q17, [%[c_ptr0], #0x10]\n" + "ldr q18, [%[c_ptr0], #0x20]\n" + "ldr q19, [%[c_ptr0], #0x30]\n" + "ldr q20, [c_ptr1]\n" + "ldr q21, [c_ptr1, #0x10]\n" + "ldr q22, [c_ptr1, #0x20]\n" + "ldr q23, [c_ptr1, #0x30]\n" + "ldr q24, [c_ptr2]\n" + "ldr q25, [c_ptr2, #0x10]\n" + "ldr q26, [c_ptr2, #0x20]\n" + "ldr q27, [c_ptr2, #0x30]\n" + "ldr q28, [c_ptr3]\n" + "ldr q29, [c_ptr3, #0x10]\n" + "ldr q30, [c_ptr3, #0x20]\n" + "ldr q31, [c_ptr3, #0x30]\n" + "ldr q0, [%[a_ptr0]]\n" "add %[a_ptr0], %[a_ptr0], #0x10\n" - "add c_ptr2, c_ptr1, %[ldc]\n" + "ldr q1, [a_ptr1]\n" "add a_ptr1, a_ptr1, #0x10\n" - "add c_ptr3, c_ptr2, %[ldc]\n" + "ldr q2, [a_ptr2]\n" "add a_ptr2, a_ptr2, #0x10\n" + "ldr q3, [a_ptr3]\n" "add a_ptr3, a_ptr3, #0x10\n" + "ldr q8, [%[b_ptr0]]\n" + "ldr q9, [%[b_ptr0], #0x10]\n" + "ldr q10, [%[b_ptr0], #0x20]\n" + "ldr q11, [%[b_ptr0], #0x30]\n" + "ldr q12, [%[b_ptr0], #0x40]\n" + "ldr q13, [%[b_ptr0], #0x50]\n" + "ldr d14, [%[b_ptr0], #0x60]\n" + "ldr temploadreg2, [%[b_ptr0], #0x68]\n" "add %[b_ptr0], %[b_ptr0], #0x80\n" - "cbz %[loops], 1f\n" - "2:\n" + "ins v14.d[1], temploadreg2\n" + "cbz %[loops], 2f\n" + "3:\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d15, [%[b_ptr0], #-0x10]\n" ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" @@ -1870,8 +1983,8 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "ins v13.d[1], temploadreg1\n" "prfm PLDL1KEEP, [a_ptr3, #0x40]\n" "ins v14.d[1], temploadreg2\n" - "b.ne 2b\n" - "1:\n" + "b.ne 3b\n" + "2:\n" "ldr d15, [%[b_ptr0], #-0x10]\n" "prfm PSTL1KEEP, [%[c_ptr0]]\n" "ldr temploadreg3, [%[b_ptr0], #-0x8]\n" @@ -1879,7 +1992,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in "prfm PSTL1KEEP, [c_ptr2]\n" "prfm PSTL1KEEP, [c_ptr3]\n" "ins v15.d[1], temploadreg3\n" - "cbz %[regs], 3f\n" + "cbz %[regs], 4f\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr d4, [%[a_ptr0]]\n" ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" @@ -2098,8 +2211,8 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n" ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n" ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n" - "b 4f\n" - "3:\n" + "b 5f\n" + "4:\n" ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n" "ldr temploadreg0, [%[b_ptr0], #0x8]\n" ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n" @@ -2189,9 +2302,9 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n" ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n" ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n" - "4:\n" - "cbz %[blocks], 5f\n" - "6:\n" + "5:\n" + "cbz %[blocks], 6f\n" + "7:\n" "ldr q8, [%[b_ptr0]]\n" "subs %[blocks], %[blocks], #0x1\n" "ldr q9, [%[b_ptr0], #0x10]\n" @@ -2222,26 +2335,26 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "b.ne 6b\n" - "5:\n" - "cbz %[odds], 7f\n" + "b.ne 7b\n" + "6:\n" + "cbz %[odds], 8f\n" "ld1 {v0.b}[0], [%[a_ptr0]], #1\n" "ld1 {v1.b}[0], [a_ptr1], #1\n" "ld1 {v2.b}[0], [a_ptr2], #1\n" "ld1 {v3.b}[0], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 8f\n" + "b.eq 9f\n" "ld1 {v0.b}[1], [%[a_ptr0]], #1\n" "ld1 {v1.b}[1], [a_ptr1], #1\n" "ld1 {v2.b}[1], [a_ptr2], #1\n" "ld1 {v3.b}[1], [a_ptr3], #1\n" "subs %[odds], %[odds], #0x1\n" - "b.eq 8f\n" + "b.eq 9f\n" "ld1 {v0.b}[2], [%[a_ptr0]]\n" "ld1 {v1.b}[2], [a_ptr1]\n" "ld1 {v2.b}[2], [a_ptr2]\n" "ld1 {v3.b}[2], [a_ptr3]\n" - "8:\n" + "9:\n" "ldr q8, [%[b_ptr0]]\n" "ldr q9, [%[b_ptr0], #0x10]\n" "ldr q10, [%[b_ptr0], #0x20]\n" @@ -2262,7 +2375,7 @@ void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, in ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n" ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n" ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n" - "7:\n" + "8:\n" "str q16, [%[c_ptr0]]\n" "str q17, [%[c_ptr0], #0x10]\n" "str q18, [%[c_ptr0], #0x20]\n" |