diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/indirect-interleaves')
42 files changed, 3541 insertions, 3526 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp index 3ad32d8a50..ffd9d4b22a 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp @@ -75,61 +75,61 @@ void interleave_block<6, 1, VLType::None, false>( for (;width>7;width-=8) { __asm __volatile ( // Load up 8 elements (2 vectors) from each of 8 sources. - "VLD1.32 {d0-d3}, [%[inptr0]]!\n" // q0=A0A1A2A3 - "VLD1.32 {d4-d7}, [%[inptr1]]!\n" // q2=B0B1B2B3 - "VLD1.32 {d8-d11}, [%[inptr2]]!\n" // q4=C0C1C2C3 - "VZIP.32 q0, q4\n" // q0=A0C0A1C1, q4 = A2C2A3C3 - "VLD1.32 {d12-d15}, [%[inptr3]]!\n" // q6=D0D1D2D3 - "VZIP.32 q2, q6\n" // q2=B0D0B1D1, q6 = B2D2B3D3 - "VLD1.32 {d16-d19}, [%[inptr4]]!\n" - "VLD1.32 {d20-d23}, [%[inptr5]]!\n" - "VZIP.32 q8, q10\n" // q8=E0F0E1F1, q10 = E2F2E3F3 + "VLD1.32 {d0-d3}, [%[inptr0]]!\n" // q0=A0A1A2A3 + "VLD1.32 {d4-d7}, [%[inptr1]]!\n" // q2=B0B1B2B3 + "VLD1.32 {d8-d11}, [%[inptr2]]!\n" // q4=C0C1C2C3 + "VZIP.32 q0, q4\n" // q0=A0C0A1C1, q4 = A2C2A3C3 + "VLD1.32 {d12-d15}, [%[inptr3]]!\n" // q6=D0D1D2D3 + "VZIP.32 q2, q6\n" // q2=B0D0B1D1, q6 = B2D2B3D3 + "VLD1.32 {d16-d19}, [%[inptr4]]!\n" + "VLD1.32 {d20-d23}, [%[inptr5]]!\n" + "VZIP.32 q8, q10\n" // q8=E0F0E1F1, q10 = E2F2E3F3 ASM_PREFETCH("[%[inptr0], #128]") - "VZIP.32 q0, q2\n" // q0 = A0B0C0D0, q2 = A1B1C1D1 + "VZIP.32 q0, q2\n" // q0 = A0B0C0D0, q2 = A1B1C1D1 // Store first elements - "VST1.32 {d0-d1}, [%[outptr]]!\n" - "VST1.32 {d16}, [%[outptr]]!\n" + "VST1.32 {d0-d1}, [%[outptr]]!\n" + "VST1.32 {d16}, [%[outptr]]!\n" - "VZIP.32 q4, q6\n" // q4 = A2B2C2D2, q6 = A3B3C3D3 + "VZIP.32 q4, q6\n" // q4 = A2B2C2D2, q6 = A3B3C3D3 // Store second elements - "VST1.32 {d4-d5}, [%[outptr]]!\n" - "VZIP.32 q1, q5\n" + "VST1.32 {d4-d5}, [%[outptr]]!\n" + "VZIP.32 q1, q5\n" ASM_PREFETCH("[%[inptr1], #128]") - "VST1.32 {d17}, [%[outptr]]!\n" - "VZIP.32 q3, q7\n" + "VST1.32 {d17}, [%[outptr]]!\n" + "VZIP.32 q3, q7\n" // Store third elements - "VZIP.32 q9, q11\n" - "VST1.32 {d8-d9}, [%[outptr]]!\n" - "VZIP.32 q1, q3\n" + "VZIP.32 q9, q11\n" + "VST1.32 {d8-d9}, [%[outptr]]!\n" + "VZIP.32 q1, q3\n" ASM_PREFETCH("[%[inptr2], #128]") - "VST1.32 {d20}, [%[outptr]]!\n" + "VST1.32 {d20}, [%[outptr]]!\n" // Store fourth elements - "VZIP.32 q5, q7\n" - "VST1.32 {d12-d13}, [%[outptr]]!\n" + "VZIP.32 q5, q7\n" + "VST1.32 {d12-d13}, [%[outptr]]!\n" ASM_PREFETCH("[%[inptr3], #128]") - "VST1.32 {d21}, [%[outptr]]!\n" + "VST1.32 {d21}, [%[outptr]]!\n" // Fifth - "VST1.32 {d2-d3}, [%[outptr]]!\n" + "VST1.32 {d2-d3}, [%[outptr]]!\n" ASM_PREFETCH("[%[inptr4], #128]") - "VST1.32 {d18}, [%[outptr]]!\n" + "VST1.32 {d18}, [%[outptr]]!\n" // Sixth - "VST1.32 {d6-d7}, [%[outptr]]!\n" + "VST1.32 {d6-d7}, [%[outptr]]!\n" ASM_PREFETCH("[%[inptr5], #128]") - "VST1.32 {d19}, [%[outptr]]!\n" + "VST1.32 {d19}, [%[outptr]]!\n" // Seventh - "VST1.32 {d10-d11}, [%[outptr]]!\n" - "VST1.32 {d22}, [%[outptr]]!\n" + "VST1.32 {d10-d11}, [%[outptr]]!\n" + "VST1.32 {d22}, [%[outptr]]!\n" // Eighth - "VST1.32 {d14-d15}, [%[outptr]]!\n" - "VST1.32 {d23}, [%[outptr]]!\n" + "VST1.32 {d14-d15}, [%[outptr]]!\n" + "VST1.32 {d23}, [%[outptr]]!\n" : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [outptr] "+r" (outptr) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp index 4d065300ae..d5a41a332d 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp @@ -34,9 +34,9 @@ void interleave_block<4, 16, VLType::None, false>( "ldr x23, [%x[in], #0x0]\n" "ldr x22, [%x[in], #0x8]\n" "cmp %x[height], #0x4\n" + "add x23, x23, %x[row_offset]\n" "ldr x21, [%x[in], #0x10]\n" "ldr x20, [%x[in], #0x18]\n" - "add x23, x23, %x[row_offset]\n" "add x22, x22, %x[row_offset]\n" "add x21, x21, %x[row_offset]\n" "add x20, x20, %x[row_offset]\n" @@ -60,12 +60,12 @@ void interleave_block<4, 16, VLType::None, false>( "ldr q19, [x23], #0x10\n" "ldr q18, [x22], #0x10\n" "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" "ldr q17, [x21], #0x10\n" "ldr q16, [x20], #0x10\n" - "cmp %x[width], #0x10\n" + "str q19, [%x[out_ptr], #0x0]\n" "prfm pldl1keep, [x23, #0x70]\n" "prfm pldl1keep, [x22, #0x70]\n" - "str q19, [%x[out_ptr], #0x0]\n" "str q18, [%x[out_ptr], #0x10]\n" "prfm pldl1keep, [x21, #0x70]\n" "prfm pldl1keep, [x20, #0x70]\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp index 1cd6523c76..35de179ed4 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 1, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset], LSL #1\n" "add x27, x27, %x[row_offset], LSL #1\n" "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset], LSL #1\n" "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset], LSL #1\n" "add x22, x22, %x[row_offset], LSL #1\n" "add x21, x21, %x[row_offset], LSL #1\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,52 +79,52 @@ void interleave_block<8, 1, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q23, [x28], #0x10\n" - "ldr q25, [x27], #0x10\n" + "ldr q25, [x28], #0x10\n" + "ldr q27, [x27], #0x10\n" "subs %x[width], %x[width], #0x8\n" - "ldr q22, [x26], #0x10\n" - "ldr q21, [x25], #0x10\n" "cmp %x[width], #0x8\n" - "ldr q19, [x24], #0x10\n" - "ldr q18, [x23], #0x10\n" + "ldr q26, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "ldr q21, [x24], #0x10\n" + "ldr q20, [x23], #0x10\n" + "zip1 v23.8h, v25.8h, v21.8h\n" + "zip1 v22.8h, v27.8h, v20.8h\n" "ldr q17, [x22], #0x10\n" "ldr q16, [x21], #0x10\n" + "zip1 v19.8h, v26.8h, v17.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" + "zip2 v25.8h, v25.8h, v21.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "zip1 v20.8h, v23.8h, v19.8h\n" - "zip1 v24.8h, v25.8h, v18.8h\n" - "zip2 v23.8h, v23.8h, v19.8h\n" - "zip2 v25.8h, v25.8h, v18.8h\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v19.8h, v22.8h, v17.8h\n" - "zip1 v18.8h, v21.8h, v16.8h\n" + "zip1 v24.8h, v23.8h, v19.8h\n" + "zip1 v17.8h, v22.8h, v18.8h\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" - "zip2 v22.8h, v22.8h, v17.8h\n" - "zip2 v17.8h, v21.8h, v16.8h\n" + "zip2 v23.8h, v23.8h, v19.8h\n" + "zip2 v19.8h, v22.8h, v18.8h\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip1 v21.8h, v20.8h, v19.8h\n" - "zip1 v16.8h, v24.8h, v18.8h\n" - "zip2 v20.8h, v20.8h, v19.8h\n" - "zip2 v19.8h, v24.8h, v18.8h\n" - "zip1 v24.8h, v23.8h, v22.8h\n" - "zip1 v18.8h, v25.8h, v17.8h\n" - "zip2 v23.8h, v23.8h, v22.8h\n" - "zip2 v22.8h, v25.8h, v17.8h\n" - "zip1 v17.8h, v21.8h, v16.8h\n" - "zip2 v16.8h, v21.8h, v16.8h\n" - "zip1 v21.8h, v20.8h, v19.8h\n" - "zip2 v20.8h, v20.8h, v19.8h\n" - "zip1 v19.8h, v24.8h, v18.8h\n" - "zip2 v18.8h, v24.8h, v18.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v22.8h, v25.8h, v21.8h\n" + "zip1 v18.8h, v20.8h, v16.8h\n" + "zip2 v21.8h, v25.8h, v21.8h\n" + "zip2 v20.8h, v20.8h, v16.8h\n" + "zip1 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v24.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x10]\n" - "zip1 v17.8h, v23.8h, v22.8h\n" - "zip2 v16.8h, v23.8h, v22.8h\n" - "str q21, [%x[out_ptr], #0x20]\n" - "str q20, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v23.8h, v19.8h\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip1 v19.8h, v22.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v18.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "zip2 v16.8h, v21.8h, v20.8h\n" "str q19, [%x[out_ptr], #0x40]\n" "str q18, [%x[out_ptr], #0x50]\n" "str q17, [%x[out_ptr], #0x60]\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp index 81c758f498..59981e9979 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 1, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset], LSL #1\n" "add x27, x27, %x[row_offset], LSL #1\n" "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset], LSL #1\n" "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset], LSL #1\n" "add x22, x22, %x[row_offset], LSL #1\n" "add x21, x21, %x[row_offset], LSL #1\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,53 +79,53 @@ void interleave_block<8, 1, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr d25, [x28], #0x8\n" - "ldr d24, [x27], #0x8\n" + "ldr d27, [x28], #0x8\n" + "ldr d26, [x27], #0x8\n" + "fcvtl v27.4s, v27.4h\n" + "fcvtl v26.4s, v26.4h\n" + "ldr d22, [x26], #0x8\n" + "ldr d21, [x25], #0x8\n" + "fcvtl v22.4s, v22.4h\n" + "fcvtl v21.4s, v21.4h\n" + "ldr d20, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "fcvtl v20.4s, v20.4h\n" + "fcvtl v25.4s, v25.4h\n" + "ldr d19, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" + "fcvtl v19.4s, v19.4h\n" + "fcvtl v16.4s, v16.4h\n" + "zip1 v24.4s, v27.4s, v22.4s\n" + "zip1 v23.4s, v26.4s, v21.4s\n" "subs %x[width], %x[width], #0x4\n" - "ldr d18, [x26], #0x8\n" - "ldr d17, [x25], #0x8\n" "cmp %x[width], #0x4\n" - "ldr d23, [x24], #0x8\n" - "ldr d22, [x23], #0x8\n" - "ldr d21, [x22], #0x8\n" - "ldr d16, [x21], #0x8\n" - "fcvtl v25.4s, v25.4h\n" - "fcvtl v24.4s, v24.4h\n" - "fcvtl v18.4s, v18.4h\n" - "fcvtl v17.4s, v17.4h\n" + "zip1 v18.4s, v20.4s, v19.4s\n" + "zip1 v17.4s, v25.4s, v16.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "fcvtl v23.4s, v23.4h\n" - "fcvtl v22.4s, v22.4h\n" + "zip2 v22.4s, v27.4s, v22.4s\n" + "zip2 v21.4s, v26.4s, v21.4s\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "fcvtl v21.4s, v21.4h\n" - "fcvtl v16.4s, v16.4h\n" + "zip2 v20.4s, v20.4s, v19.4s\n" + "zip2 v19.4s, v25.4s, v16.4s\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" - "zip1 v20.4s, v25.4s, v18.4s\n" - "zip1 v19.4s, v24.4s, v17.4s\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip2 v26.4s, v25.4s, v18.4s\n" - "zip2 v25.4s, v24.4s, v17.4s\n" - "zip1 v18.4s, v23.4s, v21.4s\n" - "zip1 v17.4s, v22.4s, v16.4s\n" - "zip2 v24.4s, v23.4s, v21.4s\n" - "zip2 v23.4s, v22.4s, v16.4s\n" - "zip1 v16.4s, v20.4s, v19.4s\n" - "zip2 v22.4s, v20.4s, v19.4s\n" - "zip1 v21.4s, v18.4s, v17.4s\n" - "zip2 v20.4s, v18.4s, v17.4s\n" - "zip1 v19.4s, v26.4s, v25.4s\n" - "zip1 v18.4s, v24.4s, v23.4s\n" + "zip1 v16.4s, v24.4s, v23.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip2 v17.4s, v26.4s, v25.4s\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" "zip2 v16.4s, v24.4s, v23.4s\n" - "str q21, [%x[out_ptr], #0x10]\n" - "str q22, [%x[out_ptr], #0x20]\n" - "str q20, [%x[out_ptr], #0x30]\n" - "str q19, [%x[out_ptr], #0x40]\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v17.4s, v18.4s, v17.4s\n" + "zip1 v16.4s, v22.4s, v21.4s\n" + "str q17, [%x[out_ptr], #0x30]\n" + "zip1 v18.4s, v20.4s, v19.4s\n" + "zip2 v17.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip2 v16.4s, v20.4s, v19.4s\n" "str q18, [%x[out_ptr], #0x50]\n" "str q17, [%x[out_ptr], #0x60]\n" "str q16, [%x[out_ptr], #0x70]\n" @@ -134,76 +134,76 @@ void interleave_block<8, 1, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 6f\n" "tbz %x[width], #1, 4f\n" - "ldr s29, [x28], #0x4\n" - "ldr s28, [x27], #0x4\n" + "ldr s28, [x28], #0x4\n" + "ldr s27, [x27], #0x4\n" "mov x20, #0x2\n" - "ldr s27, [x26], #0x4\n" - "ldr s26, [x25], #0x4\n" - "ldr s25, [x24], #0x4\n" - "ldr s24, [x23], #0x4\n" - "ldr s23, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" + "ldr s24, [x24], #0x4\n" + "ldr s23, [x23], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" "tbz %x[width], #0, 5f\n" - "ld1 { v29.h }[2], [x28]\n" - "ld1 { v28.h }[2], [x27]\n" + "ld1 { v28.h }[2], [x28]\n" + "ld1 { v27.h }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v27.h }[2], [x26]\n" - "ld1 { v26.h }[2], [x25]\n" - "ld1 { v25.h }[2], [x24]\n" - "ld1 { v24.h }[2], [x23]\n" - "ld1 { v23.h }[2], [x22]\n" - "ld1 { v22.h }[2], [x21]\n" + "ld1 { v26.h }[2], [x26]\n" + "ld1 { v25.h }[2], [x25]\n" + "ld1 { v24.h }[2], [x24]\n" + "ld1 { v23.h }[2], [x23]\n" + "ld1 { v22.h }[2], [x22]\n" + "ld1 { v21.h }[2], [x21]\n" "b 5f\n" "4:" // odd_loads_1_0 - "ldr h29, [x28, #0x0]\n" - "ldr h28, [x27, #0x0]\n" + "ldr h28, [x28, #0x0]\n" + "ldr h27, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h27, [x26, #0x0]\n" - "ldr h26, [x25, #0x0]\n" - "ldr h25, [x24, #0x0]\n" - "ldr h24, [x23, #0x0]\n" - "ldr h23, [x22, #0x0]\n" - "ldr h22, [x21, #0x0]\n" + "ldr h26, [x26, #0x0]\n" + "ldr h25, [x25, #0x0]\n" + "ldr h24, [x24, #0x0]\n" + "ldr h23, [x23, #0x0]\n" + "ldr h22, [x22, #0x0]\n" + "ldr h21, [x21, #0x0]\n" "5:" // Odd load end - "fcvtl v29.4s, v29.4h\n" "fcvtl v28.4s, v28.4h\n" - "subs x20, x20, #0x1\n" "fcvtl v27.4s, v27.4h\n" + "subs x20, x20, #0x1\n" "fcvtl v26.4s, v26.4h\n" "fcvtl v25.4s, v25.4h\n" "fcvtl v24.4s, v24.4h\n" "fcvtl v23.4s, v23.4h\n" "fcvtl v22.4s, v22.4h\n" - "zip1 v21.4s, v29.4s, v27.4s\n" + "fcvtl v21.4s, v21.4h\n" "zip1 v20.4s, v28.4s, v26.4s\n" - "zip1 v19.4s, v25.4s, v23.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" "zip1 v18.4s, v24.4s, v22.4s\n" - "zip1 v17.4s, v21.4s, v20.4s\n" - "zip1 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" "subs x20, x20, #0x1\n" - "zip2 v17.4s, v21.4s, v20.4s\n" - "zip2 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" - "zip2 v19.4s, v29.4s, v27.4s\n" - "zip2 v17.4s, v28.4s, v26.4s\n" - "zip2 v18.4s, v25.4s, v23.4s\n" - "zip2 v16.4s, v24.4s, v22.4s\n" - "zip1 v17.4s, v19.4s, v17.4s\n" - "zip1 v16.4s, v18.4s, v16.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v19.4s, v28.4s, v26.4s\n" + "zip2 v16.4s, v27.4s, v25.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v19.4s, v16.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "6:" // Odds skip : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp index 521b3bf8f5..9eeabfa9eb 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 1, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #2\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset], LSL #2\n" "add x27, x27, %x[row_offset], LSL #2\n" "add x26, x26, %x[row_offset], LSL #2\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset], LSL #2\n" "add x24, x24, %x[row_offset], LSL #2\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset], LSL #2\n" "add x22, x22, %x[row_offset], LSL #2\n" "add x21, x21, %x[row_offset], LSL #2\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,46 +79,46 @@ void interleave_block<8, 1, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q25, [x28], #0x10\n" - "ldr q24, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q18, [x27], #0x10\n" "subs %x[width], %x[width], #0x4\n" - "ldr q18, [x26], #0x10\n" - "ldr q17, [x25], #0x10\n" "cmp %x[width], #0x4\n" - "ldr q23, [x24], #0x10\n" - "ldr q22, [x23], #0x10\n" - "ldr q21, [x22], #0x10\n" + "ldr q17, [x26], #0x10\n" + "ldr q16, [x25], #0x10\n" + "zip1 v25.4s, v20.4s, v17.4s\n" + "zip1 v24.4s, v18.4s, v16.4s\n" + "ldr q19, [x24], #0x10\n" + "ldr q23, [x23], #0x10\n" + "zip2 v22.4s, v20.4s, v17.4s\n" + "zip2 v21.4s, v18.4s, v16.4s\n" + "ldr q18, [x22], #0x10\n" "ldr q16, [x21], #0x10\n" - "zip1 v20.4s, v25.4s, v18.4s\n" - "zip1 v19.4s, v24.4s, v17.4s\n" - "zip2 v26.4s, v25.4s, v18.4s\n" - "zip2 v25.4s, v24.4s, v17.4s\n" + "zip1 v20.4s, v19.4s, v18.4s\n" + "zip1 v17.4s, v23.4s, v16.4s\n" + "zip2 v19.4s, v19.4s, v18.4s\n" + "zip2 v18.4s, v23.4s, v16.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v18.4s, v23.4s, v21.4s\n" - "zip1 v17.4s, v22.4s, v16.4s\n" - "zip2 v24.4s, v23.4s, v21.4s\n" - "zip2 v23.4s, v22.4s, v16.4s\n" + "zip1 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip1 v16.4s, v20.4s, v19.4s\n" - "zip2 v22.4s, v20.4s, v19.4s\n" - "zip1 v21.4s, v18.4s, v17.4s\n" - "zip2 v20.4s, v18.4s, v17.4s\n" - "zip1 v19.4s, v26.4s, v25.4s\n" - "zip1 v18.4s, v24.4s, v23.4s\n" - "str q16, [%x[out_ptr], #0x0]\n" - "zip2 v17.4s, v26.4s, v25.4s\n" - "zip2 v16.4s, v24.4s, v23.4s\n" - "str q21, [%x[out_ptr], #0x10]\n" - "str q22, [%x[out_ptr], #0x20]\n" - "str q20, [%x[out_ptr], #0x30]\n" - "str q19, [%x[out_ptr], #0x40]\n" - "str q18, [%x[out_ptr], #0x50]\n" + "zip2 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v16.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "zip2 v17.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v16.4s, v19.4s, v18.4s\n" "str q17, [%x[out_ptr], #0x60]\n" "str q16, [%x[out_ptr], #0x70]\n" "add %x[out_ptr], %x[out_ptr], #0x80\n" @@ -126,68 +126,68 @@ void interleave_block<8, 1, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 6f\n" "tbz %x[width], #1, 4f\n" - "ldr d29, [x28], #0x8\n" - "ldr d28, [x27], #0x8\n" + "ldr d28, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" "mov x20, #0x2\n" - "ldr d27, [x26], #0x8\n" - "ldr d26, [x25], #0x8\n" - "ldr d25, [x24], #0x8\n" - "ldr d24, [x23], #0x8\n" - "ldr d23, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "ldr d24, [x24], #0x8\n" + "ldr d23, [x23], #0x8\n" + "ldr d22, [x22], #0x8\n" + "ldr d21, [x21], #0x8\n" "tbz %x[width], #0, 5f\n" - "ld1 { v29.s }[2], [x28]\n" - "ld1 { v28.s }[2], [x27]\n" + "ld1 { v28.s }[2], [x28]\n" + "ld1 { v27.s }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v27.s }[2], [x26]\n" - "ld1 { v26.s }[2], [x25]\n" - "ld1 { v25.s }[2], [x24]\n" - "ld1 { v24.s }[2], [x23]\n" - "ld1 { v23.s }[2], [x22]\n" - "ld1 { v22.s }[2], [x21]\n" + "ld1 { v26.s }[2], [x26]\n" + "ld1 { v25.s }[2], [x25]\n" + "ld1 { v24.s }[2], [x24]\n" + "ld1 { v23.s }[2], [x23]\n" + "ld1 { v22.s }[2], [x22]\n" + "ld1 { v21.s }[2], [x21]\n" "b 5f\n" "4:" // odd_loads_1_0 - "ldr s29, [x28, #0x0]\n" - "ldr s28, [x27, #0x0]\n" + "ldr s28, [x28, #0x0]\n" + "ldr s27, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr s27, [x26, #0x0]\n" - "ldr s26, [x25, #0x0]\n" - "ldr s25, [x24, #0x0]\n" - "ldr s24, [x23, #0x0]\n" - "ldr s23, [x22, #0x0]\n" - "ldr s22, [x21, #0x0]\n" + "ldr s26, [x26, #0x0]\n" + "ldr s25, [x25, #0x0]\n" + "ldr s24, [x24, #0x0]\n" + "ldr s23, [x23, #0x0]\n" + "ldr s22, [x22, #0x0]\n" + "ldr s21, [x21, #0x0]\n" "5:" // Odd load end - "zip1 v21.4s, v29.4s, v27.4s\n" "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip1 v19.4s, v25.4s, v23.4s\n" "zip1 v18.4s, v24.4s, v22.4s\n" - "zip1 v17.4s, v21.4s, v20.4s\n" - "zip1 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" "subs x20, x20, #0x1\n" - "zip2 v17.4s, v21.4s, v20.4s\n" - "zip2 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" - "zip2 v19.4s, v29.4s, v27.4s\n" - "zip2 v17.4s, v28.4s, v26.4s\n" - "zip2 v18.4s, v25.4s, v23.4s\n" - "zip2 v16.4s, v24.4s, v22.4s\n" - "zip1 v17.4s, v19.4s, v17.4s\n" - "zip1 v16.4s, v18.4s, v16.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v19.4s, v28.4s, v26.4s\n" + "zip2 v16.4s, v27.4s, v25.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v19.4s, v16.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "6:" // Odds skip : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp index 8f67a21d05..27b3335694 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 1, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset], LSL #1\n" "add x27, x27, %x[row_offset], LSL #1\n" "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset], LSL #1\n" "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset], LSL #1\n" "add x22, x22, %x[row_offset], LSL #1\n" "add x21, x21, %x[row_offset], LSL #1\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,52 +79,52 @@ void interleave_block<8, 1, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q23, [x28], #0x10\n" - "ldr q25, [x27], #0x10\n" + "ldr q25, [x28], #0x10\n" + "ldr q27, [x27], #0x10\n" "subs %x[width], %x[width], #0x8\n" - "ldr q22, [x26], #0x10\n" - "ldr q21, [x25], #0x10\n" "cmp %x[width], #0x8\n" - "ldr q19, [x24], #0x10\n" - "ldr q18, [x23], #0x10\n" + "ldr q26, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "ldr q21, [x24], #0x10\n" + "ldr q20, [x23], #0x10\n" + "zip1 v23.8h, v25.8h, v21.8h\n" + "zip1 v22.8h, v27.8h, v20.8h\n" "ldr q17, [x22], #0x10\n" "ldr q16, [x21], #0x10\n" + "zip1 v19.8h, v26.8h, v17.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" + "zip2 v25.8h, v25.8h, v21.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "zip1 v20.8h, v23.8h, v19.8h\n" - "zip1 v24.8h, v25.8h, v18.8h\n" - "zip2 v23.8h, v23.8h, v19.8h\n" - "zip2 v25.8h, v25.8h, v18.8h\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v19.8h, v22.8h, v17.8h\n" - "zip1 v18.8h, v21.8h, v16.8h\n" + "zip1 v24.8h, v23.8h, v19.8h\n" + "zip1 v17.8h, v22.8h, v18.8h\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" - "zip2 v22.8h, v22.8h, v17.8h\n" - "zip2 v17.8h, v21.8h, v16.8h\n" + "zip2 v23.8h, v23.8h, v19.8h\n" + "zip2 v19.8h, v22.8h, v18.8h\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip1 v21.8h, v20.8h, v19.8h\n" - "zip1 v16.8h, v24.8h, v18.8h\n" - "zip2 v20.8h, v20.8h, v19.8h\n" - "zip2 v19.8h, v24.8h, v18.8h\n" - "zip1 v24.8h, v23.8h, v22.8h\n" - "zip1 v18.8h, v25.8h, v17.8h\n" - "zip2 v23.8h, v23.8h, v22.8h\n" - "zip2 v22.8h, v25.8h, v17.8h\n" - "zip1 v17.8h, v21.8h, v16.8h\n" - "zip2 v16.8h, v21.8h, v16.8h\n" - "zip1 v21.8h, v20.8h, v19.8h\n" - "zip2 v20.8h, v20.8h, v19.8h\n" - "zip1 v19.8h, v24.8h, v18.8h\n" - "zip2 v18.8h, v24.8h, v18.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v22.8h, v25.8h, v21.8h\n" + "zip1 v18.8h, v20.8h, v16.8h\n" + "zip2 v21.8h, v25.8h, v21.8h\n" + "zip2 v20.8h, v20.8h, v16.8h\n" + "zip1 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v24.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x10]\n" - "zip1 v17.8h, v23.8h, v22.8h\n" - "zip2 v16.8h, v23.8h, v22.8h\n" - "str q21, [%x[out_ptr], #0x20]\n" - "str q20, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v23.8h, v19.8h\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip1 v19.8h, v22.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v18.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "zip2 v16.8h, v21.8h, v20.8h\n" "str q19, [%x[out_ptr], #0x40]\n" "str q18, [%x[out_ptr], #0x50]\n" "str q17, [%x[out_ptr], #0x60]\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp index 0a76fa812e..4c90691edc 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 1, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset]\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset]\n" "add x27, x27, %x[row_offset]\n" "add x26, x26, %x[row_offset]\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset]\n" "add x24, x24, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset]\n" "add x22, x22, %x[row_offset]\n" "add x21, x21, %x[row_offset]\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,60 +79,60 @@ void interleave_block<8, 1, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr d23, [x28], #0x8\n" - "ldr d25, [x27], #0x8\n" - "subs %x[width], %x[width], #0x8\n" - "ldr d22, [x26], #0x8\n" - "ldr d21, [x25], #0x8\n" - "cmp %x[width], #0x8\n" - "ldr d19, [x24], #0x8\n" - "ldr d18, [x23], #0x8\n" - "ldr d17, [x22], #0x8\n" - "ldr d16, [x21], #0x8\n" - "sshll v23.8h, v23.8b, #0x0\n" + "ldr d25, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" "sshll v25.8h, v25.8b, #0x0\n" - "sshll v22.8h, v22.8b, #0x0\n" + "sshll v27.8h, v27.8b, #0x0\n" + "ldr d26, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "sshll v26.8h, v26.8b, #0x0\n" + "sshll v24.8h, v24.8b, #0x0\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" "sshll v21.8h, v21.8b, #0x0\n" + "sshll v20.8h, v20.8b, #0x0\n" + "ldr d17, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" + "sshll v17.8h, v17.8b, #0x0\n" + "sshll v16.8h, v16.8b, #0x0\n" + "zip1 v23.8h, v25.8h, v21.8h\n" + "zip1 v22.8h, v26.8h, v17.8h\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "zip1 v19.8h, v27.8h, v20.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "sshll v19.8h, v19.8b, #0x0\n" - "sshll v18.8h, v18.8b, #0x0\n" + "zip2 v25.8h, v25.8h, v21.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "sshll v17.8h, v17.8b, #0x0\n" - "sshll v16.8h, v16.8b, #0x0\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v24.8h, v23.8h, v22.8h\n" + "zip1 v17.8h, v19.8h, v18.8h\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip1 v20.8h, v23.8h, v19.8h\n" - "zip1 v24.8h, v25.8h, v18.8h\n" - "zip2 v23.8h, v23.8h, v19.8h\n" - "zip2 v25.8h, v25.8h, v18.8h\n" - "zip1 v19.8h, v22.8h, v17.8h\n" - "zip1 v18.8h, v21.8h, v16.8h\n" - "zip2 v22.8h, v22.8h, v17.8h\n" - "zip2 v17.8h, v21.8h, v16.8h\n" - "zip1 v21.8h, v20.8h, v19.8h\n" - "zip1 v16.8h, v24.8h, v18.8h\n" - "zip2 v20.8h, v20.8h, v19.8h\n" - "zip2 v19.8h, v24.8h, v18.8h\n" - "zip1 v24.8h, v23.8h, v22.8h\n" - "zip1 v18.8h, v25.8h, v17.8h\n" "zip2 v23.8h, v23.8h, v22.8h\n" - "zip2 v22.8h, v25.8h, v17.8h\n" - "zip1 v17.8h, v21.8h, v16.8h\n" - "zip2 v16.8h, v21.8h, v16.8h\n" - "zip1 v21.8h, v20.8h, v19.8h\n" - "zip2 v20.8h, v20.8h, v19.8h\n" - "zip1 v19.8h, v24.8h, v18.8h\n" - "zip2 v18.8h, v24.8h, v18.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v19.8h, v19.8h, v18.8h\n" + "zip1 v22.8h, v25.8h, v21.8h\n" + "zip1 v18.8h, v20.8h, v16.8h\n" + "zip2 v21.8h, v25.8h, v21.8h\n" + "zip2 v20.8h, v20.8h, v16.8h\n" + "zip1 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v24.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x10]\n" - "zip1 v17.8h, v23.8h, v22.8h\n" - "zip2 v16.8h, v23.8h, v22.8h\n" - "str q21, [%x[out_ptr], #0x20]\n" - "str q20, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v23.8h, v19.8h\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip1 v19.8h, v22.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v18.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "zip2 v16.8h, v21.8h, v20.8h\n" "str q19, [%x[out_ptr], #0x40]\n" "str q18, [%x[out_ptr], #0x50]\n" "str q17, [%x[out_ptr], #0x60]\n" @@ -227,11 +227,11 @@ void interleave_block<8, 1, VLType::None, false>( "sshll v24.8h, v24.8b, #0x0\n" "sshll v23.8h, v23.8b, #0x0\n" "zip1 v22.8h, v30.8h, v26.8h\n" - "zip1 v21.8h, v29.8h, v25.8h\n" - "zip1 v20.8h, v28.8h, v24.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" "zip1 v19.8h, v27.8h, v23.8h\n" - "zip1 v18.8h, v22.8h, v20.8h\n" - "zip1 v17.8h, v21.8h, v19.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" @@ -241,8 +241,8 @@ void interleave_block<8, 1, VLType::None, false>( "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v18.8h, v22.8h, v20.8h\n" - "zip2 v17.8h, v21.8h, v19.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp index be6e8980f6..8901908140 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 1, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset]\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset]\n" "add x27, x27, %x[row_offset]\n" "add x26, x26, %x[row_offset]\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset]\n" "add x24, x24, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset]\n" "add x22, x22, %x[row_offset]\n" "add x21, x21, %x[row_offset]\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,60 +79,60 @@ void interleave_block<8, 1, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr d23, [x28], #0x8\n" - "ldr d25, [x27], #0x8\n" - "subs %x[width], %x[width], #0x8\n" - "ldr d22, [x26], #0x8\n" - "ldr d21, [x25], #0x8\n" - "cmp %x[width], #0x8\n" - "ldr d19, [x24], #0x8\n" - "ldr d18, [x23], #0x8\n" - "ldr d17, [x22], #0x8\n" - "ldr d16, [x21], #0x8\n" - "ushll v23.8h, v23.8b, #0x0\n" + "ldr d25, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" "ushll v25.8h, v25.8b, #0x0\n" - "ushll v22.8h, v22.8b, #0x0\n" + "ushll v27.8h, v27.8b, #0x0\n" + "ldr d26, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "ushll v26.8h, v26.8b, #0x0\n" + "ushll v24.8h, v24.8b, #0x0\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" "ushll v21.8h, v21.8b, #0x0\n" + "ushll v20.8h, v20.8b, #0x0\n" + "ldr d17, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" + "ushll v17.8h, v17.8b, #0x0\n" + "ushll v16.8h, v16.8b, #0x0\n" + "zip1 v23.8h, v25.8h, v21.8h\n" + "zip1 v22.8h, v26.8h, v17.8h\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "zip1 v19.8h, v27.8h, v20.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "ushll v19.8h, v19.8b, #0x0\n" - "ushll v18.8h, v18.8b, #0x0\n" + "zip2 v25.8h, v25.8h, v21.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "ushll v17.8h, v17.8b, #0x0\n" - "ushll v16.8h, v16.8b, #0x0\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v24.8h, v23.8h, v22.8h\n" + "zip1 v17.8h, v19.8h, v18.8h\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip1 v20.8h, v23.8h, v19.8h\n" - "zip1 v24.8h, v25.8h, v18.8h\n" - "zip2 v23.8h, v23.8h, v19.8h\n" - "zip2 v25.8h, v25.8h, v18.8h\n" - "zip1 v19.8h, v22.8h, v17.8h\n" - "zip1 v18.8h, v21.8h, v16.8h\n" - "zip2 v22.8h, v22.8h, v17.8h\n" - "zip2 v17.8h, v21.8h, v16.8h\n" - "zip1 v21.8h, v20.8h, v19.8h\n" - "zip1 v16.8h, v24.8h, v18.8h\n" - "zip2 v20.8h, v20.8h, v19.8h\n" - "zip2 v19.8h, v24.8h, v18.8h\n" - "zip1 v24.8h, v23.8h, v22.8h\n" - "zip1 v18.8h, v25.8h, v17.8h\n" "zip2 v23.8h, v23.8h, v22.8h\n" - "zip2 v22.8h, v25.8h, v17.8h\n" - "zip1 v17.8h, v21.8h, v16.8h\n" - "zip2 v16.8h, v21.8h, v16.8h\n" - "zip1 v21.8h, v20.8h, v19.8h\n" - "zip2 v20.8h, v20.8h, v19.8h\n" - "zip1 v19.8h, v24.8h, v18.8h\n" - "zip2 v18.8h, v24.8h, v18.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v19.8h, v19.8h, v18.8h\n" + "zip1 v22.8h, v25.8h, v21.8h\n" + "zip1 v18.8h, v20.8h, v16.8h\n" + "zip2 v21.8h, v25.8h, v21.8h\n" + "zip2 v20.8h, v20.8h, v16.8h\n" + "zip1 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v24.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x10]\n" - "zip1 v17.8h, v23.8h, v22.8h\n" - "zip2 v16.8h, v23.8h, v22.8h\n" - "str q21, [%x[out_ptr], #0x20]\n" - "str q20, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v23.8h, v19.8h\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip1 v19.8h, v22.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v18.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "zip2 v16.8h, v21.8h, v20.8h\n" "str q19, [%x[out_ptr], #0x40]\n" "str q18, [%x[out_ptr], #0x50]\n" "str q17, [%x[out_ptr], #0x60]\n" @@ -227,11 +227,11 @@ void interleave_block<8, 1, VLType::None, false>( "ushll v24.8h, v24.8b, #0x0\n" "ushll v23.8h, v23.8b, #0x0\n" "zip1 v22.8h, v30.8h, v26.8h\n" - "zip1 v21.8h, v29.8h, v25.8h\n" - "zip1 v20.8h, v28.8h, v24.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" "zip1 v19.8h, v27.8h, v23.8h\n" - "zip1 v18.8h, v22.8h, v20.8h\n" - "zip1 v17.8h, v21.8h, v19.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" @@ -241,8 +241,8 @@ void interleave_block<8, 1, VLType::None, false>( "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v18.8h, v22.8h, v20.8h\n" - "zip2 v17.8h, v21.8h, v19.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp index f034b2b45c..db610427a8 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 2, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset], LSL #1\n" "add x27, x27, %x[row_offset], LSL #1\n" "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset], LSL #1\n" "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset], LSL #1\n" "add x22, x22, %x[row_offset], LSL #1\n" "add x21, x21, %x[row_offset], LSL #1\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,46 +79,46 @@ void interleave_block<8, 2, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q25, [x28], #0x10\n" - "ldr q24, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q18, [x27], #0x10\n" "subs %x[width], %x[width], #0x8\n" - "ldr q18, [x26], #0x10\n" - "ldr q17, [x25], #0x10\n" "cmp %x[width], #0x8\n" - "ldr q23, [x24], #0x10\n" - "ldr q22, [x23], #0x10\n" - "ldr q21, [x22], #0x10\n" + "ldr q17, [x26], #0x10\n" + "ldr q16, [x25], #0x10\n" + "zip1 v25.4s, v20.4s, v17.4s\n" + "zip1 v24.4s, v18.4s, v16.4s\n" + "ldr q19, [x24], #0x10\n" + "ldr q23, [x23], #0x10\n" + "zip2 v22.4s, v20.4s, v17.4s\n" + "zip2 v21.4s, v18.4s, v16.4s\n" + "ldr q18, [x22], #0x10\n" "ldr q16, [x21], #0x10\n" - "zip1 v20.4s, v25.4s, v18.4s\n" - "zip1 v19.4s, v24.4s, v17.4s\n" - "zip2 v26.4s, v25.4s, v18.4s\n" - "zip2 v25.4s, v24.4s, v17.4s\n" + "zip1 v20.4s, v19.4s, v18.4s\n" + "zip1 v17.4s, v23.4s, v16.4s\n" + "zip2 v19.4s, v19.4s, v18.4s\n" + "zip2 v18.4s, v23.4s, v16.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v18.4s, v23.4s, v21.4s\n" - "zip1 v17.4s, v22.4s, v16.4s\n" - "zip2 v24.4s, v23.4s, v21.4s\n" - "zip2 v23.4s, v22.4s, v16.4s\n" + "zip1 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip1 v16.4s, v20.4s, v19.4s\n" - "zip2 v22.4s, v20.4s, v19.4s\n" - "zip1 v21.4s, v18.4s, v17.4s\n" - "zip2 v20.4s, v18.4s, v17.4s\n" - "zip1 v19.4s, v26.4s, v25.4s\n" - "zip1 v18.4s, v24.4s, v23.4s\n" - "str q16, [%x[out_ptr], #0x0]\n" - "zip2 v17.4s, v26.4s, v25.4s\n" - "zip2 v16.4s, v24.4s, v23.4s\n" - "str q21, [%x[out_ptr], #0x10]\n" - "str q22, [%x[out_ptr], #0x20]\n" - "str q20, [%x[out_ptr], #0x30]\n" - "str q19, [%x[out_ptr], #0x40]\n" - "str q18, [%x[out_ptr], #0x50]\n" + "zip2 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v16.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "zip2 v17.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v16.4s, v19.4s, v18.4s\n" "str q17, [%x[out_ptr], #0x60]\n" "str q16, [%x[out_ptr], #0x70]\n" "add %x[out_ptr], %x[out_ptr], #0x80\n" @@ -126,119 +126,119 @@ void interleave_block<8, 2, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 8f\n" "tbz %x[width], #2, 5f\n" - "ldr d29, [x28], #0x8\n" - "ldr d28, [x27], #0x8\n" - "ldr d27, [x26], #0x8\n" - "ldr d26, [x25], #0x8\n" - "ldr d25, [x24], #0x8\n" - "ldr d24, [x23], #0x8\n" - "ldr d23, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d28, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "ldr d24, [x24], #0x8\n" + "ldr d23, [x23], #0x8\n" + "ldr d22, [x22], #0x8\n" + "ldr d21, [x21], #0x8\n" "tbz %x[width], #1, 4f\n" - "ld1 { v29.s }[2], [x28], #0x4\n" - "ld1 { v28.s }[2], [x27], #0x4\n" + "ld1 { v28.s }[2], [x28], #0x4\n" + "ld1 { v27.s }[2], [x27], #0x4\n" "mov x20, #0x3\n" - "ld1 { v27.s }[2], [x26], #0x4\n" - "ld1 { v26.s }[2], [x25], #0x4\n" - "ld1 { v25.s }[2], [x24], #0x4\n" - "ld1 { v24.s }[2], [x23], #0x4\n" - "ld1 { v23.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v26.s }[2], [x26], #0x4\n" + "ld1 { v25.s }[2], [x25], #0x4\n" + "ld1 { v24.s }[2], [x24], #0x4\n" + "ld1 { v23.s }[2], [x23], #0x4\n" + "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v21.s }[2], [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v29.h }[6], [x28]\n" - "ld1 { v28.h }[6], [x27]\n" + "ld1 { v28.h }[6], [x28]\n" + "ld1 { v27.h }[6], [x27]\n" "mov x20, #0x4\n" - "ld1 { v27.h }[6], [x26]\n" - "ld1 { v26.h }[6], [x25]\n" - "ld1 { v25.h }[6], [x24]\n" - "ld1 { v24.h }[6], [x23]\n" - "ld1 { v23.h }[6], [x22]\n" - "ld1 { v22.h }[6], [x21]\n" + "ld1 { v26.h }[6], [x26]\n" + "ld1 { v25.h }[6], [x25]\n" + "ld1 { v24.h }[6], [x24]\n" + "ld1 { v23.h }[6], [x23]\n" + "ld1 { v22.h }[6], [x22]\n" + "ld1 { v21.h }[6], [x21]\n" "b 7f\n" "4:" // odd_loads_1_4 "mov x20, #0x2\n" "tbz %x[width], #0, 7f\n" - "ld1 { v29.h }[4], [x28]\n" - "ld1 { v28.h }[4], [x27]\n" + "ld1 { v28.h }[4], [x28]\n" + "ld1 { v27.h }[4], [x27]\n" "mov x20, #0x3\n" - "ld1 { v27.h }[4], [x26]\n" - "ld1 { v26.h }[4], [x25]\n" - "ld1 { v25.h }[4], [x24]\n" - "ld1 { v24.h }[4], [x23]\n" - "ld1 { v23.h }[4], [x22]\n" - "ld1 { v22.h }[4], [x21]\n" + "ld1 { v26.h }[4], [x26]\n" + "ld1 { v25.h }[4], [x25]\n" + "ld1 { v24.h }[4], [x24]\n" + "ld1 { v23.h }[4], [x23]\n" + "ld1 { v22.h }[4], [x22]\n" + "ld1 { v21.h }[4], [x21]\n" "b 7f\n" "5:" // odd_loads_2_0 "tbz %x[width], #1, 6f\n" - "ldr s29, [x28], #0x4\n" - "ldr s28, [x27], #0x4\n" + "ldr s28, [x28], #0x4\n" + "ldr s27, [x27], #0x4\n" "mov x20, #0x1\n" - "ldr s27, [x26], #0x4\n" - "ldr s26, [x25], #0x4\n" - "ldr s25, [x24], #0x4\n" - "ldr s24, [x23], #0x4\n" - "ldr s23, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" + "ldr s24, [x24], #0x4\n" + "ldr s23, [x23], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v29.h }[2], [x28]\n" - "ld1 { v28.h }[2], [x27]\n" + "ld1 { v28.h }[2], [x28]\n" + "ld1 { v27.h }[2], [x27]\n" "mov x20, #0x2\n" - "ld1 { v27.h }[2], [x26]\n" - "ld1 { v26.h }[2], [x25]\n" - "ld1 { v25.h }[2], [x24]\n" - "ld1 { v24.h }[2], [x23]\n" - "ld1 { v23.h }[2], [x22]\n" - "ld1 { v22.h }[2], [x21]\n" + "ld1 { v26.h }[2], [x26]\n" + "ld1 { v25.h }[2], [x25]\n" + "ld1 { v24.h }[2], [x24]\n" + "ld1 { v23.h }[2], [x23]\n" + "ld1 { v22.h }[2], [x22]\n" + "ld1 { v21.h }[2], [x21]\n" "b 7f\n" "6:" // odd_loads_1_0 - "ldr h29, [x28, #0x0]\n" - "ldr h28, [x27, #0x0]\n" + "ldr h28, [x28, #0x0]\n" + "ldr h27, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h27, [x26, #0x0]\n" - "ldr h26, [x25, #0x0]\n" - "ldr h25, [x24, #0x0]\n" - "ldr h24, [x23, #0x0]\n" - "ldr h23, [x22, #0x0]\n" - "ldr h22, [x21, #0x0]\n" + "ldr h26, [x26, #0x0]\n" + "ldr h25, [x25, #0x0]\n" + "ldr h24, [x24, #0x0]\n" + "ldr h23, [x23, #0x0]\n" + "ldr h22, [x22, #0x0]\n" + "ldr h21, [x21, #0x0]\n" "7:" // Odd load end - "zip1 v21.4s, v29.4s, v27.4s\n" "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip1 v19.4s, v25.4s, v23.4s\n" "zip1 v18.4s, v24.4s, v22.4s\n" - "zip1 v17.4s, v21.4s, v20.4s\n" - "zip1 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v17.4s, v21.4s, v20.4s\n" - "zip2 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 8f\n" - "zip2 v21.4s, v29.4s, v27.4s\n" "zip2 v20.4s, v28.4s, v26.4s\n" + "zip2 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip2 v19.4s, v25.4s, v23.4s\n" "zip2 v18.4s, v24.4s, v22.4s\n" - "zip1 v17.4s, v21.4s, v20.4s\n" - "zip1 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 8f\n" - "zip2 v17.4s, v21.4s, v20.4s\n" - "zip2 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "8:" // Odds skip : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp index a57810ce20..b3a52451e8 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 2, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #2\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset], LSL #2\n" "add x27, x27, %x[row_offset], LSL #2\n" "add x26, x26, %x[row_offset], LSL #2\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset], LSL #2\n" "add x24, x24, %x[row_offset], LSL #2\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset], LSL #2\n" "add x22, x22, %x[row_offset], LSL #2\n" "add x21, x21, %x[row_offset], LSL #2\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,100 +79,100 @@ void interleave_block<8, 2, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q21, [x28], #0x10\n" - "ldr q16, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" "subs %x[width], %x[width], #0x4\n" - "ldr q20, [x26], #0x10\n" - "ldr q19, [x25], #0x10\n" "cmp %x[width], #0x4\n" - "ldr q24, [x24], #0x10\n" - "ldr q18, [x23], #0x10\n" - "ldr q23, [x22], #0x10\n" - "ldr q22, [x21], #0x10\n" - "zip1 v17.2d, v21.2d, v16.2d\n" - "zip2 v21.2d, v21.2d, v16.2d\n" + "ldr q25, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "zip1 v16.2d, v20.2d, v19.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" + "ldr q23, [x24], #0x10\n" + "ldr q22, [x23], #0x10\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip2 v21.2d, v20.2d, v19.2d\n" + "ldr q20, [x22], #0x10\n" + "ldr q19, [x21], #0x10\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v20.2d, v19.2d\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "zip1 v16.2d, v20.2d, v19.2d\n" - "zip2 v20.2d, v20.2d, v19.2d\n" + "str q18, [%x[out_ptr], #0x10]\n" + "zip2 v18.2d, v25.2d, v24.2d\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v19.2d, v24.2d, v18.2d\n" - "zip2 v18.2d, v24.2d, v18.2d\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip2 v17.2d, v23.2d, v22.2d\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" - "str q17, [%x[out_ptr], #0x0]\n" - "zip1 v17.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip2 v16.2d, v20.2d, v19.2d\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "str q16, [%x[out_ptr], #0x10]\n" - "zip2 v16.2d, v23.2d, v22.2d\n" - "str q19, [%x[out_ptr], #0x20]\n" - "str q17, [%x[out_ptr], #0x30]\n" "str q21, [%x[out_ptr], #0x40]\n" - "str q20, [%x[out_ptr], #0x50]\n" - "str q18, [%x[out_ptr], #0x60]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" "str q16, [%x[out_ptr], #0x70]\n" "add %x[out_ptr], %x[out_ptr], #0x80\n" "bge 2b\n" "3:" // Main loop skip "cbz %x[width], 6f\n" "tbz %x[width], #1, 4f\n" - "ldr d27, [x28], #0x8\n" - "ldr d26, [x27], #0x8\n" + "ldr d25, [x28], #0x8\n" + "ldr d24, [x27], #0x8\n" "mov x20, #0x1\n" - "ldr d25, [x26], #0x8\n" - "ldr d24, [x25], #0x8\n" - "ldr d23, [x24], #0x8\n" - "ldr d22, [x23], #0x8\n" - "ldr d21, [x22], #0x8\n" - "ldr d20, [x21], #0x8\n" + "ldr d23, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d19, [x22], #0x8\n" + "ldr d18, [x21], #0x8\n" "tbz %x[width], #0, 5f\n" - "ld1 { v27.s }[2], [x28]\n" - "ld1 { v26.s }[2], [x27]\n" + "ld1 { v25.s }[2], [x28]\n" + "ld1 { v24.s }[2], [x27]\n" "mov x20, #0x2\n" - "ld1 { v25.s }[2], [x26]\n" - "ld1 { v24.s }[2], [x25]\n" - "ld1 { v23.s }[2], [x24]\n" - "ld1 { v22.s }[2], [x23]\n" - "ld1 { v21.s }[2], [x22]\n" - "ld1 { v20.s }[2], [x21]\n" + "ld1 { v23.s }[2], [x26]\n" + "ld1 { v22.s }[2], [x25]\n" + "ld1 { v21.s }[2], [x24]\n" + "ld1 { v20.s }[2], [x23]\n" + "ld1 { v19.s }[2], [x22]\n" + "ld1 { v18.s }[2], [x21]\n" "b 5f\n" "4:" // odd_loads_1_0 - "ldr s27, [x28, #0x0]\n" - "ldr s26, [x27, #0x0]\n" + "ldr s25, [x28, #0x0]\n" + "ldr s24, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr s25, [x26, #0x0]\n" - "ldr s24, [x25, #0x0]\n" - "ldr s23, [x24, #0x0]\n" - "ldr s22, [x23, #0x0]\n" - "ldr s21, [x22, #0x0]\n" - "ldr s20, [x21, #0x0]\n" + "ldr s23, [x26, #0x0]\n" + "ldr s22, [x25, #0x0]\n" + "ldr s21, [x24, #0x0]\n" + "ldr s20, [x23, #0x0]\n" + "ldr s19, [x22, #0x0]\n" + "ldr s18, [x21, #0x0]\n" "5:" // Odd load end "subs x20, x20, #0x1\n" - "zip1 v19.2d, v27.2d, v26.2d\n" - "zip1 v18.2d, v25.2d, v24.2d\n" - "zip1 v17.2d, v23.2d, v22.2d\n" - "zip1 v16.2d, v21.2d, v20.2d\n" - "str q19, [%x[out_ptr], #0x0]\n" - "str q18, [%x[out_ptr], #0x10]\n" + "zip1 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.2d, v21.2d, v20.2d\n" + "zip1 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "beq 6f\n" - "zip2 v19.2d, v27.2d, v26.2d\n" - "zip2 v18.2d, v25.2d, v24.2d\n" - "zip2 v17.2d, v23.2d, v22.2d\n" - "zip2 v16.2d, v21.2d, v20.2d\n" - "str q19, [%x[out_ptr], #0x0]\n" - "str q18, [%x[out_ptr], #0x10]\n" + "zip2 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v17.2d, v21.2d, v20.2d\n" + "zip2 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "6:" // Odds skip : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp index edc1375b02..33639c201e 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 4, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset], LSL #1\n" "add x27, x27, %x[row_offset], LSL #1\n" "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset], LSL #1\n" "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset], LSL #1\n" "add x22, x22, %x[row_offset], LSL #1\n" "add x21, x21, %x[row_offset], LSL #1\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,142 +79,142 @@ void interleave_block<8, 4, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q21, [x28], #0x10\n" - "ldr q16, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" "subs %x[width], %x[width], #0x8\n" - "ldr q20, [x26], #0x10\n" - "ldr q19, [x25], #0x10\n" "cmp %x[width], #0x8\n" - "ldr q24, [x24], #0x10\n" - "ldr q18, [x23], #0x10\n" - "ldr q23, [x22], #0x10\n" - "ldr q22, [x21], #0x10\n" - "zip1 v17.2d, v21.2d, v16.2d\n" - "zip2 v21.2d, v21.2d, v16.2d\n" + "ldr q25, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "zip1 v16.2d, v20.2d, v19.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" + "ldr q23, [x24], #0x10\n" + "ldr q22, [x23], #0x10\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip2 v21.2d, v20.2d, v19.2d\n" + "ldr q20, [x22], #0x10\n" + "ldr q19, [x21], #0x10\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v20.2d, v19.2d\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "zip1 v16.2d, v20.2d, v19.2d\n" - "zip2 v20.2d, v20.2d, v19.2d\n" + "str q18, [%x[out_ptr], #0x10]\n" + "zip2 v18.2d, v25.2d, v24.2d\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v19.2d, v24.2d, v18.2d\n" - "zip2 v18.2d, v24.2d, v18.2d\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip2 v17.2d, v23.2d, v22.2d\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" - "str q17, [%x[out_ptr], #0x0]\n" - "zip1 v17.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip2 v16.2d, v20.2d, v19.2d\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "str q16, [%x[out_ptr], #0x10]\n" - "zip2 v16.2d, v23.2d, v22.2d\n" - "str q19, [%x[out_ptr], #0x20]\n" - "str q17, [%x[out_ptr], #0x30]\n" "str q21, [%x[out_ptr], #0x40]\n" - "str q20, [%x[out_ptr], #0x50]\n" - "str q18, [%x[out_ptr], #0x60]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" "str q16, [%x[out_ptr], #0x70]\n" "add %x[out_ptr], %x[out_ptr], #0x80\n" "bge 2b\n" "3:" // Main loop skip "cbz %x[width], 8f\n" "tbz %x[width], #2, 5f\n" - "ldr d27, [x28], #0x8\n" - "ldr d26, [x27], #0x8\n" - "ldr d25, [x26], #0x8\n" - "ldr d24, [x25], #0x8\n" - "ldr d23, [x24], #0x8\n" - "ldr d22, [x23], #0x8\n" - "ldr d21, [x22], #0x8\n" - "ldr d20, [x21], #0x8\n" + "ldr d25, [x28], #0x8\n" + "ldr d24, [x27], #0x8\n" + "ldr d23, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d19, [x22], #0x8\n" + "ldr d18, [x21], #0x8\n" "tbz %x[width], #1, 4f\n" - "ld1 { v27.s }[2], [x28], #0x4\n" - "ld1 { v26.s }[2], [x27], #0x4\n" + "ld1 { v25.s }[2], [x28], #0x4\n" + "ld1 { v24.s }[2], [x27], #0x4\n" "mov x20, #0x2\n" - "ld1 { v25.s }[2], [x26], #0x4\n" - "ld1 { v24.s }[2], [x25], #0x4\n" - "ld1 { v23.s }[2], [x24], #0x4\n" - "ld1 { v22.s }[2], [x23], #0x4\n" - "ld1 { v21.s }[2], [x22], #0x4\n" - "ld1 { v20.s }[2], [x21], #0x4\n" + "ld1 { v23.s }[2], [x26], #0x4\n" + "ld1 { v22.s }[2], [x25], #0x4\n" + "ld1 { v21.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v19.s }[2], [x22], #0x4\n" + "ld1 { v18.s }[2], [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v27.h }[6], [x28]\n" - "ld1 { v26.h }[6], [x27]\n" - "ld1 { v25.h }[6], [x26]\n" - "ld1 { v24.h }[6], [x25]\n" - "ld1 { v23.h }[6], [x24]\n" - "ld1 { v22.h }[6], [x23]\n" - "ld1 { v21.h }[6], [x22]\n" - "ld1 { v20.h }[6], [x21]\n" + "ld1 { v25.h }[6], [x28]\n" + "ld1 { v24.h }[6], [x27]\n" + "ld1 { v23.h }[6], [x26]\n" + "ld1 { v22.h }[6], [x25]\n" + "ld1 { v21.h }[6], [x24]\n" + "ld1 { v20.h }[6], [x23]\n" + "ld1 { v19.h }[6], [x22]\n" + "ld1 { v18.h }[6], [x21]\n" "b 7f\n" "4:" // odd_loads_1_4 "mov x20, #0x1\n" "tbz %x[width], #0, 7f\n" - "ld1 { v27.h }[4], [x28]\n" - "ld1 { v26.h }[4], [x27]\n" + "ld1 { v25.h }[4], [x28]\n" + "ld1 { v24.h }[4], [x27]\n" "mov x20, #0x2\n" - "ld1 { v25.h }[4], [x26]\n" - "ld1 { v24.h }[4], [x25]\n" - "ld1 { v23.h }[4], [x24]\n" - "ld1 { v22.h }[4], [x23]\n" - "ld1 { v21.h }[4], [x22]\n" - "ld1 { v20.h }[4], [x21]\n" + "ld1 { v23.h }[4], [x26]\n" + "ld1 { v22.h }[4], [x25]\n" + "ld1 { v21.h }[4], [x24]\n" + "ld1 { v20.h }[4], [x23]\n" + "ld1 { v19.h }[4], [x22]\n" + "ld1 { v18.h }[4], [x21]\n" "b 7f\n" "5:" // odd_loads_2_0 "tbz %x[width], #1, 6f\n" - "ldr s27, [x28], #0x4\n" - "ldr s26, [x27], #0x4\n" + "ldr s25, [x28], #0x4\n" + "ldr s24, [x27], #0x4\n" "mov x20, #0x1\n" - "ldr s25, [x26], #0x4\n" - "ldr s24, [x25], #0x4\n" - "ldr s23, [x24], #0x4\n" - "ldr s22, [x23], #0x4\n" - "ldr s21, [x22], #0x4\n" - "ldr s20, [x21], #0x4\n" + "ldr s23, [x26], #0x4\n" + "ldr s22, [x25], #0x4\n" + "ldr s21, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s19, [x22], #0x4\n" + "ldr s18, [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v27.h }[2], [x28]\n" - "ld1 { v26.h }[2], [x27]\n" - "ld1 { v25.h }[2], [x26]\n" - "ld1 { v24.h }[2], [x25]\n" - "ld1 { v23.h }[2], [x24]\n" - "ld1 { v22.h }[2], [x23]\n" - "ld1 { v21.h }[2], [x22]\n" - "ld1 { v20.h }[2], [x21]\n" + "ld1 { v25.h }[2], [x28]\n" + "ld1 { v24.h }[2], [x27]\n" + "ld1 { v23.h }[2], [x26]\n" + "ld1 { v22.h }[2], [x25]\n" + "ld1 { v21.h }[2], [x24]\n" + "ld1 { v20.h }[2], [x23]\n" + "ld1 { v19.h }[2], [x22]\n" + "ld1 { v18.h }[2], [x21]\n" "b 7f\n" "6:" // odd_loads_1_0 - "ldr h27, [x28, #0x0]\n" - "ldr h26, [x27, #0x0]\n" + "ldr h25, [x28, #0x0]\n" + "ldr h24, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h25, [x26, #0x0]\n" - "ldr h24, [x25, #0x0]\n" - "ldr h23, [x24, #0x0]\n" - "ldr h22, [x23, #0x0]\n" - "ldr h21, [x22, #0x0]\n" - "ldr h20, [x21, #0x0]\n" + "ldr h23, [x26, #0x0]\n" + "ldr h22, [x25, #0x0]\n" + "ldr h21, [x24, #0x0]\n" + "ldr h20, [x23, #0x0]\n" + "ldr h19, [x22, #0x0]\n" + "ldr h18, [x21, #0x0]\n" "7:" // Odd load end "subs x20, x20, #0x1\n" - "zip1 v19.2d, v27.2d, v26.2d\n" - "zip1 v18.2d, v25.2d, v24.2d\n" - "zip1 v17.2d, v23.2d, v22.2d\n" - "zip1 v16.2d, v21.2d, v20.2d\n" - "str q19, [%x[out_ptr], #0x0]\n" - "str q18, [%x[out_ptr], #0x10]\n" + "zip1 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.2d, v21.2d, v20.2d\n" + "zip1 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "beq 8f\n" - "zip2 v19.2d, v27.2d, v26.2d\n" - "zip2 v18.2d, v25.2d, v24.2d\n" - "zip2 v17.2d, v23.2d, v22.2d\n" - "zip2 v16.2d, v21.2d, v20.2d\n" - "str q19, [%x[out_ptr], #0x0]\n" - "str q18, [%x[out_ptr], #0x10]\n" + "zip2 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v17.2d, v21.2d, v20.2d\n" + "zip2 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "8:" // Odds skip : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp index ef1493b605..3044cfde48 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 4, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #2\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset], LSL #2\n" "add x27, x27, %x[row_offset], LSL #2\n" "add x26, x26, %x[row_offset], LSL #2\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset], LSL #2\n" "add x24, x24, %x[row_offset], LSL #2\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset], LSL #2\n" "add x22, x22, %x[row_offset], LSL #2\n" "add x21, x21, %x[row_offset], LSL #2\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,36 +79,36 @@ void interleave_block<8, 4, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q19, [x28], #0x10\n" - "ldr q18, [x26], #0x10\n" - "subs %x[width], %x[width], #0x4\n" + "ldr q17, [x28], #0x10\n" + "ldr q16, [x26], #0x10\n" + ".inst 0x0ea16a37 // bfcvtn v23.4h, v17.4s\n" + ".inst 0x0ea16a16 // bfcvtn v22.4h, v16.4s\n" "ldr q17, [x24], #0x10\n" "ldr q16, [x22], #0x10\n" + ".inst 0x0ea16a35 // bfcvtn v21.4h, v17.4s\n" + ".inst 0x0ea16a14 // bfcvtn v20.4h, v16.4s\n" + "ldr q19, [x27], #0x10\n" + "ldr q18, [x25], #0x10\n" + "subs %x[width], %x[width], #0x4\n" "cmp %x[width], #0x4\n" - "ldr q23, [x27], #0x10\n" - "ldr q22, [x25], #0x10\n" - "ldr q21, [x23], #0x10\n" - "ldr q20, [x21], #0x10\n" - ".inst 0x0ea16a73 // bfcvtn v19.4h, v19.4s\n" - ".inst 0x0ea16a52 // bfcvtn v18.4h, v18.4s\n" - ".inst 0x0ea16a31 // bfcvtn v17.4h, v17.4s\n" - ".inst 0x0ea16a10 // bfcvtn v16.4h, v16.4s\n" + "ldr q17, [x23], #0x10\n" + "ldr q16, [x21], #0x10\n" + ".inst 0x4ea16a77 // bfcvtn2 v23.8h, v19.4s\n" + ".inst 0x4ea16a56 // bfcvtn2 v22.8h, v18.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" + ".inst 0x4ea16a35 // bfcvtn2 v21.8h, v17.4s\n" + ".inst 0x4ea16a14 // bfcvtn2 v20.8h, v16.4s\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" + "str q23, [%x[out_ptr], #0x0]\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" - ".inst 0x4ea16af3 // bfcvtn2 v19.8h, v23.4s\n" - ".inst 0x4ea16ad2 // bfcvtn2 v18.8h, v22.4s\n" + "str q22, [%x[out_ptr], #0x10]\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - ".inst 0x4ea16ab1 // bfcvtn2 v17.8h, v21.4s\n" - ".inst 0x4ea16a90 // bfcvtn2 v16.8h, v20.4s\n" - "str q19, [%x[out_ptr], #0x0]\n" - "str q18, [%x[out_ptr], #0x10]\n" - "str q17, [%x[out_ptr], #0x20]\n" - "str q16, [%x[out_ptr], #0x30]\n" + "str q21, [%x[out_ptr], #0x20]\n" + "str q20, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "bge 2b\n" "3:" // Main loop skip @@ -150,9 +150,9 @@ void interleave_block<8, 4, VLType::None, false>( ".inst 0x0ea16a10 // bfcvtn v16.4h, v16.4s\n" ".inst 0x4ea16af3 // bfcvtn2 v19.8h, v23.4s\n" ".inst 0x4ea16ad2 // bfcvtn2 v18.8h, v22.4s\n" + "str q19, [%x[out_ptr], #0x0]\n" ".inst 0x4ea16ab1 // bfcvtn2 v17.8h, v21.4s\n" ".inst 0x4ea16a90 // bfcvtn2 v16.8h, v20.4s\n" - "str q19, [%x[out_ptr], #0x0]\n" "str q18, [%x[out_ptr], #0x10]\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp index ad213db3e5..8bb9898861 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 4, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset]\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset]\n" "add x27, x27, %x[row_offset]\n" "add x26, x26, %x[row_offset]\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset]\n" "add x24, x24, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset]\n" "add x22, x22, %x[row_offset]\n" "add x21, x21, %x[row_offset]\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,46 +79,46 @@ void interleave_block<8, 4, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q25, [x28], #0x10\n" - "ldr q24, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q18, [x27], #0x10\n" "subs %x[width], %x[width], #0x10\n" - "ldr q18, [x26], #0x10\n" - "ldr q17, [x25], #0x10\n" "cmp %x[width], #0x10\n" - "ldr q23, [x24], #0x10\n" - "ldr q22, [x23], #0x10\n" - "ldr q21, [x22], #0x10\n" + "ldr q17, [x26], #0x10\n" + "ldr q16, [x25], #0x10\n" + "zip1 v25.4s, v20.4s, v17.4s\n" + "zip1 v24.4s, v18.4s, v16.4s\n" + "ldr q19, [x24], #0x10\n" + "ldr q23, [x23], #0x10\n" + "zip2 v22.4s, v20.4s, v17.4s\n" + "zip2 v21.4s, v18.4s, v16.4s\n" + "ldr q18, [x22], #0x10\n" "ldr q16, [x21], #0x10\n" - "zip1 v20.4s, v25.4s, v18.4s\n" - "zip1 v19.4s, v24.4s, v17.4s\n" - "zip2 v26.4s, v25.4s, v18.4s\n" - "zip2 v25.4s, v24.4s, v17.4s\n" + "zip1 v20.4s, v19.4s, v18.4s\n" + "zip1 v17.4s, v23.4s, v16.4s\n" + "zip2 v19.4s, v19.4s, v18.4s\n" + "zip2 v18.4s, v23.4s, v16.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v18.4s, v23.4s, v21.4s\n" - "zip1 v17.4s, v22.4s, v16.4s\n" - "zip2 v24.4s, v23.4s, v21.4s\n" - "zip2 v23.4s, v22.4s, v16.4s\n" + "zip1 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip1 v16.4s, v20.4s, v19.4s\n" - "zip2 v22.4s, v20.4s, v19.4s\n" - "zip1 v21.4s, v18.4s, v17.4s\n" - "zip2 v20.4s, v18.4s, v17.4s\n" - "zip1 v19.4s, v26.4s, v25.4s\n" - "zip1 v18.4s, v24.4s, v23.4s\n" - "str q16, [%x[out_ptr], #0x0]\n" - "zip2 v17.4s, v26.4s, v25.4s\n" - "zip2 v16.4s, v24.4s, v23.4s\n" - "str q21, [%x[out_ptr], #0x10]\n" - "str q22, [%x[out_ptr], #0x20]\n" - "str q20, [%x[out_ptr], #0x30]\n" - "str q19, [%x[out_ptr], #0x40]\n" - "str q18, [%x[out_ptr], #0x50]\n" + "zip2 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v16.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "zip2 v17.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v16.4s, v19.4s, v18.4s\n" "str q17, [%x[out_ptr], #0x60]\n" "str q16, [%x[out_ptr], #0x70]\n" "add %x[out_ptr], %x[out_ptr], #0x80\n" @@ -126,203 +126,203 @@ void interleave_block<8, 4, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 12f\n" "tbz %x[width], #3, 7f\n" - "ldr d29, [x28], #0x8\n" - "ldr d28, [x27], #0x8\n" - "ldr d27, [x26], #0x8\n" - "ldr d26, [x25], #0x8\n" - "ldr d25, [x24], #0x8\n" - "ldr d24, [x23], #0x8\n" - "ldr d23, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d28, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "ldr d24, [x24], #0x8\n" + "ldr d23, [x23], #0x8\n" + "ldr d22, [x22], #0x8\n" + "ldr d21, [x21], #0x8\n" "tbz %x[width], #2, 5f\n" - "ld1 { v29.s }[2], [x28], #0x4\n" - "ld1 { v28.s }[2], [x27], #0x4\n" - "ld1 { v27.s }[2], [x26], #0x4\n" - "ld1 { v26.s }[2], [x25], #0x4\n" - "ld1 { v25.s }[2], [x24], #0x4\n" - "ld1 { v24.s }[2], [x23], #0x4\n" - "ld1 { v23.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v28.s }[2], [x28], #0x4\n" + "ld1 { v27.s }[2], [x27], #0x4\n" + "ld1 { v26.s }[2], [x26], #0x4\n" + "ld1 { v25.s }[2], [x25], #0x4\n" + "ld1 { v24.s }[2], [x24], #0x4\n" + "ld1 { v23.s }[2], [x23], #0x4\n" + "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v21.s }[2], [x21], #0x4\n" "tbz %x[width], #1, 4f\n" - "ld1 { v29.h }[6], [x28], #0x2\n" - "ld1 { v28.h }[6], [x27], #0x2\n" + "ld1 { v28.h }[6], [x28], #0x2\n" + "ld1 { v27.h }[6], [x27], #0x2\n" "mov x20, #0x4\n" - "ld1 { v27.h }[6], [x26], #0x2\n" - "ld1 { v26.h }[6], [x25], #0x2\n" - "ld1 { v25.h }[6], [x24], #0x2\n" - "ld1 { v24.h }[6], [x23], #0x2\n" - "ld1 { v23.h }[6], [x22], #0x2\n" - "ld1 { v22.h }[6], [x21], #0x2\n" + "ld1 { v26.h }[6], [x26], #0x2\n" + "ld1 { v25.h }[6], [x25], #0x2\n" + "ld1 { v24.h }[6], [x24], #0x2\n" + "ld1 { v23.h }[6], [x23], #0x2\n" + "ld1 { v22.h }[6], [x22], #0x2\n" + "ld1 { v21.h }[6], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v29.b }[14], [x28]\n" - "ld1 { v28.b }[14], [x27]\n" - "ld1 { v27.b }[14], [x26]\n" - "ld1 { v26.b }[14], [x25]\n" - "ld1 { v25.b }[14], [x24]\n" - "ld1 { v24.b }[14], [x23]\n" - "ld1 { v23.b }[14], [x22]\n" - "ld1 { v22.b }[14], [x21]\n" + "ld1 { v28.b }[14], [x28]\n" + "ld1 { v27.b }[14], [x27]\n" + "ld1 { v26.b }[14], [x26]\n" + "ld1 { v25.b }[14], [x25]\n" + "ld1 { v24.b }[14], [x24]\n" + "ld1 { v23.b }[14], [x23]\n" + "ld1 { v22.b }[14], [x22]\n" + "ld1 { v21.b }[14], [x21]\n" "b 11f\n" "4:" // odd_loads_1_12 "mov x20, #0x3\n" "tbz %x[width], #0, 11f\n" - "ld1 { v29.b }[12], [x28]\n" - "ld1 { v28.b }[12], [x27]\n" + "ld1 { v28.b }[12], [x28]\n" + "ld1 { v27.b }[12], [x27]\n" "mov x20, #0x4\n" - "ld1 { v27.b }[12], [x26]\n" - "ld1 { v26.b }[12], [x25]\n" - "ld1 { v25.b }[12], [x24]\n" - "ld1 { v24.b }[12], [x23]\n" - "ld1 { v23.b }[12], [x22]\n" - "ld1 { v22.b }[12], [x21]\n" + "ld1 { v26.b }[12], [x26]\n" + "ld1 { v25.b }[12], [x25]\n" + "ld1 { v24.b }[12], [x24]\n" + "ld1 { v23.b }[12], [x23]\n" + "ld1 { v22.b }[12], [x22]\n" + "ld1 { v21.b }[12], [x21]\n" "b 11f\n" "5:" // odd_loads_2_8 "tbz %x[width], #1, 6f\n" - "ld1 { v29.h }[4], [x28], #0x2\n" - "ld1 { v28.h }[4], [x27], #0x2\n" + "ld1 { v28.h }[4], [x28], #0x2\n" + "ld1 { v27.h }[4], [x27], #0x2\n" "mov x20, #0x3\n" - "ld1 { v27.h }[4], [x26], #0x2\n" - "ld1 { v26.h }[4], [x25], #0x2\n" - "ld1 { v25.h }[4], [x24], #0x2\n" - "ld1 { v24.h }[4], [x23], #0x2\n" - "ld1 { v23.h }[4], [x22], #0x2\n" - "ld1 { v22.h }[4], [x21], #0x2\n" + "ld1 { v26.h }[4], [x26], #0x2\n" + "ld1 { v25.h }[4], [x25], #0x2\n" + "ld1 { v24.h }[4], [x24], #0x2\n" + "ld1 { v23.h }[4], [x23], #0x2\n" + "ld1 { v22.h }[4], [x22], #0x2\n" + "ld1 { v21.h }[4], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v29.b }[10], [x28]\n" - "ld1 { v28.b }[10], [x27]\n" - "ld1 { v27.b }[10], [x26]\n" - "ld1 { v26.b }[10], [x25]\n" - "ld1 { v25.b }[10], [x24]\n" - "ld1 { v24.b }[10], [x23]\n" - "ld1 { v23.b }[10], [x22]\n" - "ld1 { v22.b }[10], [x21]\n" + "ld1 { v28.b }[10], [x28]\n" + "ld1 { v27.b }[10], [x27]\n" + "ld1 { v26.b }[10], [x26]\n" + "ld1 { v25.b }[10], [x25]\n" + "ld1 { v24.b }[10], [x24]\n" + "ld1 { v23.b }[10], [x23]\n" + "ld1 { v22.b }[10], [x22]\n" + "ld1 { v21.b }[10], [x21]\n" "b 11f\n" "6:" // odd_loads_1_8 "mov x20, #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v29.b }[8], [x28]\n" - "ld1 { v28.b }[8], [x27]\n" + "ld1 { v28.b }[8], [x28]\n" + "ld1 { v27.b }[8], [x27]\n" "mov x20, #0x3\n" - "ld1 { v27.b }[8], [x26]\n" - "ld1 { v26.b }[8], [x25]\n" - "ld1 { v25.b }[8], [x24]\n" - "ld1 { v24.b }[8], [x23]\n" - "ld1 { v23.b }[8], [x22]\n" - "ld1 { v22.b }[8], [x21]\n" + "ld1 { v26.b }[8], [x26]\n" + "ld1 { v25.b }[8], [x25]\n" + "ld1 { v24.b }[8], [x24]\n" + "ld1 { v23.b }[8], [x23]\n" + "ld1 { v22.b }[8], [x22]\n" + "ld1 { v21.b }[8], [x21]\n" "b 11f\n" "7:" // odd_loads_4_0 "tbz %x[width], #2, 9f\n" - "ldr s29, [x28], #0x4\n" - "ldr s28, [x27], #0x4\n" - "ldr s27, [x26], #0x4\n" - "ldr s26, [x25], #0x4\n" - "ldr s25, [x24], #0x4\n" - "ldr s24, [x23], #0x4\n" - "ldr s23, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s28, [x28], #0x4\n" + "ldr s27, [x27], #0x4\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" + "ldr s24, [x24], #0x4\n" + "ldr s23, [x23], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" "tbz %x[width], #1, 8f\n" - "ld1 { v29.h }[2], [x28], #0x2\n" - "ld1 { v28.h }[2], [x27], #0x2\n" + "ld1 { v28.h }[2], [x28], #0x2\n" + "ld1 { v27.h }[2], [x27], #0x2\n" "mov x20, #0x2\n" - "ld1 { v27.h }[2], [x26], #0x2\n" - "ld1 { v26.h }[2], [x25], #0x2\n" - "ld1 { v25.h }[2], [x24], #0x2\n" - "ld1 { v24.h }[2], [x23], #0x2\n" - "ld1 { v23.h }[2], [x22], #0x2\n" - "ld1 { v22.h }[2], [x21], #0x2\n" + "ld1 { v26.h }[2], [x26], #0x2\n" + "ld1 { v25.h }[2], [x25], #0x2\n" + "ld1 { v24.h }[2], [x24], #0x2\n" + "ld1 { v23.h }[2], [x23], #0x2\n" + "ld1 { v22.h }[2], [x22], #0x2\n" + "ld1 { v21.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v29.b }[6], [x28]\n" - "ld1 { v28.b }[6], [x27]\n" - "ld1 { v27.b }[6], [x26]\n" - "ld1 { v26.b }[6], [x25]\n" - "ld1 { v25.b }[6], [x24]\n" - "ld1 { v24.b }[6], [x23]\n" - "ld1 { v23.b }[6], [x22]\n" - "ld1 { v22.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x28]\n" + "ld1 { v27.b }[6], [x27]\n" + "ld1 { v26.b }[6], [x26]\n" + "ld1 { v25.b }[6], [x25]\n" + "ld1 { v24.b }[6], [x24]\n" + "ld1 { v23.b }[6], [x23]\n" + "ld1 { v22.b }[6], [x22]\n" + "ld1 { v21.b }[6], [x21]\n" "b 11f\n" "8:" // odd_loads_1_4 "mov x20, #0x1\n" "tbz %x[width], #0, 11f\n" - "ld1 { v29.b }[4], [x28]\n" - "ld1 { v28.b }[4], [x27]\n" + "ld1 { v28.b }[4], [x28]\n" + "ld1 { v27.b }[4], [x27]\n" "mov x20, #0x2\n" - "ld1 { v27.b }[4], [x26]\n" - "ld1 { v26.b }[4], [x25]\n" - "ld1 { v25.b }[4], [x24]\n" - "ld1 { v24.b }[4], [x23]\n" - "ld1 { v23.b }[4], [x22]\n" - "ld1 { v22.b }[4], [x21]\n" + "ld1 { v26.b }[4], [x26]\n" + "ld1 { v25.b }[4], [x25]\n" + "ld1 { v24.b }[4], [x24]\n" + "ld1 { v23.b }[4], [x23]\n" + "ld1 { v22.b }[4], [x22]\n" + "ld1 { v21.b }[4], [x21]\n" "b 11f\n" "9:" // odd_loads_2_0 "tbz %x[width], #1, 10f\n" - "ldr h29, [x28], #0x2\n" - "ldr h28, [x27], #0x2\n" + "ldr h28, [x28], #0x2\n" + "ldr h27, [x27], #0x2\n" "mov x20, #0x1\n" - "ldr h27, [x26], #0x2\n" - "ldr h26, [x25], #0x2\n" - "ldr h25, [x24], #0x2\n" - "ldr h24, [x23], #0x2\n" - "ldr h23, [x22], #0x2\n" - "ldr h22, [x21], #0x2\n" + "ldr h26, [x26], #0x2\n" + "ldr h25, [x25], #0x2\n" + "ldr h24, [x24], #0x2\n" + "ldr h23, [x23], #0x2\n" + "ldr h22, [x22], #0x2\n" + "ldr h21, [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v29.b }[2], [x28]\n" - "ld1 { v28.b }[2], [x27]\n" - "ld1 { v27.b }[2], [x26]\n" - "ld1 { v26.b }[2], [x25]\n" - "ld1 { v25.b }[2], [x24]\n" - "ld1 { v24.b }[2], [x23]\n" - "ld1 { v23.b }[2], [x22]\n" - "ld1 { v22.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x28]\n" + "ld1 { v27.b }[2], [x27]\n" + "ld1 { v26.b }[2], [x26]\n" + "ld1 { v25.b }[2], [x25]\n" + "ld1 { v24.b }[2], [x24]\n" + "ld1 { v23.b }[2], [x23]\n" + "ld1 { v22.b }[2], [x22]\n" + "ld1 { v21.b }[2], [x21]\n" "b 11f\n" "10:" // odd_loads_1_0 - "ldr b29, [x28, #0x0]\n" - "ldr b28, [x27, #0x0]\n" + "ldr b28, [x28, #0x0]\n" + "ldr b27, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b27, [x26, #0x0]\n" - "ldr b26, [x25, #0x0]\n" - "ldr b25, [x24, #0x0]\n" - "ldr b24, [x23, #0x0]\n" - "ldr b23, [x22, #0x0]\n" - "ldr b22, [x21, #0x0]\n" + "ldr b26, [x26, #0x0]\n" + "ldr b25, [x25, #0x0]\n" + "ldr b24, [x24, #0x0]\n" + "ldr b23, [x23, #0x0]\n" + "ldr b22, [x22, #0x0]\n" + "ldr b21, [x21, #0x0]\n" "11:" // Odd load end - "zip1 v21.4s, v29.4s, v27.4s\n" "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip1 v19.4s, v25.4s, v23.4s\n" "zip1 v18.4s, v24.4s, v22.4s\n" - "zip1 v17.4s, v21.4s, v20.4s\n" - "zip1 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 12f\n" "subs x20, x20, #0x1\n" - "zip2 v17.4s, v21.4s, v20.4s\n" - "zip2 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 12f\n" - "zip2 v21.4s, v29.4s, v27.4s\n" "zip2 v20.4s, v28.4s, v26.4s\n" + "zip2 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip2 v19.4s, v25.4s, v23.4s\n" "zip2 v18.4s, v24.4s, v22.4s\n" - "zip1 v17.4s, v21.4s, v20.4s\n" - "zip1 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 12f\n" - "zip2 v17.4s, v21.4s, v20.4s\n" - "zip2 v16.4s, v19.4s, v18.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "12:" // Odds skip : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp index 28d2196ade..6e1efa3814 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp @@ -34,29 +34,29 @@ void interleave_block<8, 8, VLType::None, false>( "ldr x28, [%x[in], #0x0]\n" "ldr x27, [%x[in], #0x8]\n" "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset]\n" "ldr x26, [%x[in], #0x10]\n" "ldr x25, [%x[in], #0x18]\n" - "ldr x24, [%x[in], #0x20]\n" - "ldr x23, [%x[in], #0x28]\n" - "ldr x22, [%x[in], #0x30]\n" - "ldr x21, [%x[in], #0x38]\n" - "add x28, x28, %x[row_offset]\n" "add x27, x27, %x[row_offset]\n" "add x26, x26, %x[row_offset]\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" "add x25, x25, %x[row_offset]\n" "add x24, x24, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" "add x23, x23, %x[row_offset]\n" "add x22, x22, %x[row_offset]\n" "add x21, x21, %x[row_offset]\n" "beq 1f\n" "cmp %x[height], #0x2\n" - "mov x21, x28\n" "csel x27, x27, x28, GE\n" "csel x26, x26, x28, GT\n" "cmp %x[height], #0x4\n" "csel x25, x25, x28, GE\n" "csel x24, x24, x28, GT\n" "cmp %x[height], #0x6\n" + "mov x21, x28\n" "csel x23, x23, x28, GE\n" "csel x22, x22, x28, GT\n" "1:" // no_pointer_adj @@ -79,226 +79,226 @@ void interleave_block<8, 8, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q21, [x28], #0x10\n" - "ldr q16, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" "subs %x[width], %x[width], #0x10\n" - "ldr q20, [x26], #0x10\n" - "ldr q19, [x25], #0x10\n" "cmp %x[width], #0x10\n" - "ldr q24, [x24], #0x10\n" - "ldr q18, [x23], #0x10\n" - "ldr q23, [x22], #0x10\n" - "ldr q22, [x21], #0x10\n" - "zip1 v17.2d, v21.2d, v16.2d\n" - "zip2 v21.2d, v21.2d, v16.2d\n" + "ldr q25, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "zip1 v16.2d, v20.2d, v19.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" + "ldr q23, [x24], #0x10\n" + "ldr q22, [x23], #0x10\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip2 v21.2d, v20.2d, v19.2d\n" + "ldr q20, [x22], #0x10\n" + "ldr q19, [x21], #0x10\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v20.2d, v19.2d\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "zip1 v16.2d, v20.2d, v19.2d\n" - "zip2 v20.2d, v20.2d, v19.2d\n" + "str q18, [%x[out_ptr], #0x10]\n" + "zip2 v18.2d, v25.2d, v24.2d\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v19.2d, v24.2d, v18.2d\n" - "zip2 v18.2d, v24.2d, v18.2d\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip2 v17.2d, v23.2d, v22.2d\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" - "str q17, [%x[out_ptr], #0x0]\n" - "zip1 v17.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip2 v16.2d, v20.2d, v19.2d\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "str q16, [%x[out_ptr], #0x10]\n" - "zip2 v16.2d, v23.2d, v22.2d\n" - "str q19, [%x[out_ptr], #0x20]\n" - "str q17, [%x[out_ptr], #0x30]\n" "str q21, [%x[out_ptr], #0x40]\n" - "str q20, [%x[out_ptr], #0x50]\n" - "str q18, [%x[out_ptr], #0x60]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" "str q16, [%x[out_ptr], #0x70]\n" "add %x[out_ptr], %x[out_ptr], #0x80\n" "bge 2b\n" "3:" // Main loop skip "cbz %x[width], 12f\n" "tbz %x[width], #3, 7f\n" - "ldr d27, [x28], #0x8\n" - "ldr d26, [x27], #0x8\n" - "ldr d25, [x26], #0x8\n" - "ldr d24, [x25], #0x8\n" - "ldr d23, [x24], #0x8\n" - "ldr d22, [x23], #0x8\n" - "ldr d21, [x22], #0x8\n" - "ldr d20, [x21], #0x8\n" + "ldr d25, [x28], #0x8\n" + "ldr d24, [x27], #0x8\n" + "ldr d23, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d19, [x22], #0x8\n" + "ldr d18, [x21], #0x8\n" "tbz %x[width], #2, 5f\n" - "ld1 { v27.s }[2], [x28], #0x4\n" - "ld1 { v26.s }[2], [x27], #0x4\n" - "ld1 { v25.s }[2], [x26], #0x4\n" - "ld1 { v24.s }[2], [x25], #0x4\n" - "ld1 { v23.s }[2], [x24], #0x4\n" - "ld1 { v22.s }[2], [x23], #0x4\n" - "ld1 { v21.s }[2], [x22], #0x4\n" - "ld1 { v20.s }[2], [x21], #0x4\n" + "ld1 { v25.s }[2], [x28], #0x4\n" + "ld1 { v24.s }[2], [x27], #0x4\n" + "ld1 { v23.s }[2], [x26], #0x4\n" + "ld1 { v22.s }[2], [x25], #0x4\n" + "ld1 { v21.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v19.s }[2], [x22], #0x4\n" + "ld1 { v18.s }[2], [x21], #0x4\n" "tbz %x[width], #1, 4f\n" - "ld1 { v27.h }[6], [x28], #0x2\n" - "ld1 { v26.h }[6], [x27], #0x2\n" + "ld1 { v25.h }[6], [x28], #0x2\n" + "ld1 { v24.h }[6], [x27], #0x2\n" "mov x20, #0x2\n" - "ld1 { v25.h }[6], [x26], #0x2\n" - "ld1 { v24.h }[6], [x25], #0x2\n" - "ld1 { v23.h }[6], [x24], #0x2\n" - "ld1 { v22.h }[6], [x23], #0x2\n" - "ld1 { v21.h }[6], [x22], #0x2\n" - "ld1 { v20.h }[6], [x21], #0x2\n" + "ld1 { v23.h }[6], [x26], #0x2\n" + "ld1 { v22.h }[6], [x25], #0x2\n" + "ld1 { v21.h }[6], [x24], #0x2\n" + "ld1 { v20.h }[6], [x23], #0x2\n" + "ld1 { v19.h }[6], [x22], #0x2\n" + "ld1 { v18.h }[6], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v27.b }[14], [x28]\n" - "ld1 { v26.b }[14], [x27]\n" - "ld1 { v25.b }[14], [x26]\n" - "ld1 { v24.b }[14], [x25]\n" - "ld1 { v23.b }[14], [x24]\n" - "ld1 { v22.b }[14], [x23]\n" - "ld1 { v21.b }[14], [x22]\n" - "ld1 { v20.b }[14], [x21]\n" + "ld1 { v25.b }[14], [x28]\n" + "ld1 { v24.b }[14], [x27]\n" + "ld1 { v23.b }[14], [x26]\n" + "ld1 { v22.b }[14], [x25]\n" + "ld1 { v21.b }[14], [x24]\n" + "ld1 { v20.b }[14], [x23]\n" + "ld1 { v19.b }[14], [x22]\n" + "ld1 { v18.b }[14], [x21]\n" "b 11f\n" "4:" // odd_loads_1_12 "mov x20, #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v27.b }[12], [x28]\n" - "ld1 { v26.b }[12], [x27]\n" - "ld1 { v25.b }[12], [x26]\n" - "ld1 { v24.b }[12], [x25]\n" - "ld1 { v23.b }[12], [x24]\n" - "ld1 { v22.b }[12], [x23]\n" - "ld1 { v21.b }[12], [x22]\n" - "ld1 { v20.b }[12], [x21]\n" + "ld1 { v25.b }[12], [x28]\n" + "ld1 { v24.b }[12], [x27]\n" + "ld1 { v23.b }[12], [x26]\n" + "ld1 { v22.b }[12], [x25]\n" + "ld1 { v21.b }[12], [x24]\n" + "ld1 { v20.b }[12], [x23]\n" + "ld1 { v19.b }[12], [x22]\n" + "ld1 { v18.b }[12], [x21]\n" "b 11f\n" "5:" // odd_loads_2_8 "tbz %x[width], #1, 6f\n" - "ld1 { v27.h }[4], [x28], #0x2\n" - "ld1 { v26.h }[4], [x27], #0x2\n" + "ld1 { v25.h }[4], [x28], #0x2\n" + "ld1 { v24.h }[4], [x27], #0x2\n" "mov x20, #0x2\n" - "ld1 { v25.h }[4], [x26], #0x2\n" - "ld1 { v24.h }[4], [x25], #0x2\n" - "ld1 { v23.h }[4], [x24], #0x2\n" - "ld1 { v22.h }[4], [x23], #0x2\n" - "ld1 { v21.h }[4], [x22], #0x2\n" - "ld1 { v20.h }[4], [x21], #0x2\n" + "ld1 { v23.h }[4], [x26], #0x2\n" + "ld1 { v22.h }[4], [x25], #0x2\n" + "ld1 { v21.h }[4], [x24], #0x2\n" + "ld1 { v20.h }[4], [x23], #0x2\n" + "ld1 { v19.h }[4], [x22], #0x2\n" + "ld1 { v18.h }[4], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v27.b }[10], [x28]\n" - "ld1 { v26.b }[10], [x27]\n" - "ld1 { v25.b }[10], [x26]\n" - "ld1 { v24.b }[10], [x25]\n" - "ld1 { v23.b }[10], [x24]\n" - "ld1 { v22.b }[10], [x23]\n" - "ld1 { v21.b }[10], [x22]\n" - "ld1 { v20.b }[10], [x21]\n" + "ld1 { v25.b }[10], [x28]\n" + "ld1 { v24.b }[10], [x27]\n" + "ld1 { v23.b }[10], [x26]\n" + "ld1 { v22.b }[10], [x25]\n" + "ld1 { v21.b }[10], [x24]\n" + "ld1 { v20.b }[10], [x23]\n" + "ld1 { v19.b }[10], [x22]\n" + "ld1 { v18.b }[10], [x21]\n" "b 11f\n" "6:" // odd_loads_1_8 "mov x20, #0x1\n" "tbz %x[width], #0, 11f\n" - "ld1 { v27.b }[8], [x28]\n" - "ld1 { v26.b }[8], [x27]\n" + "ld1 { v25.b }[8], [x28]\n" + "ld1 { v24.b }[8], [x27]\n" "mov x20, #0x2\n" - "ld1 { v25.b }[8], [x26]\n" - "ld1 { v24.b }[8], [x25]\n" - "ld1 { v23.b }[8], [x24]\n" - "ld1 { v22.b }[8], [x23]\n" - "ld1 { v21.b }[8], [x22]\n" - "ld1 { v20.b }[8], [x21]\n" + "ld1 { v23.b }[8], [x26]\n" + "ld1 { v22.b }[8], [x25]\n" + "ld1 { v21.b }[8], [x24]\n" + "ld1 { v20.b }[8], [x23]\n" + "ld1 { v19.b }[8], [x22]\n" + "ld1 { v18.b }[8], [x21]\n" "b 11f\n" "7:" // odd_loads_4_0 "tbz %x[width], #2, 9f\n" - "ldr s27, [x28], #0x4\n" - "ldr s26, [x27], #0x4\n" - "ldr s25, [x26], #0x4\n" - "ldr s24, [x25], #0x4\n" - "ldr s23, [x24], #0x4\n" - "ldr s22, [x23], #0x4\n" - "ldr s21, [x22], #0x4\n" - "ldr s20, [x21], #0x4\n" + "ldr s25, [x28], #0x4\n" + "ldr s24, [x27], #0x4\n" + "ldr s23, [x26], #0x4\n" + "ldr s22, [x25], #0x4\n" + "ldr s21, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s19, [x22], #0x4\n" + "ldr s18, [x21], #0x4\n" "tbz %x[width], #1, 8f\n" - "ld1 { v27.h }[2], [x28], #0x2\n" - "ld1 { v26.h }[2], [x27], #0x2\n" + "ld1 { v25.h }[2], [x28], #0x2\n" + "ld1 { v24.h }[2], [x27], #0x2\n" "mov x20, #0x1\n" - "ld1 { v25.h }[2], [x26], #0x2\n" - "ld1 { v24.h }[2], [x25], #0x2\n" - "ld1 { v23.h }[2], [x24], #0x2\n" - "ld1 { v22.h }[2], [x23], #0x2\n" - "ld1 { v21.h }[2], [x22], #0x2\n" - "ld1 { v20.h }[2], [x21], #0x2\n" + "ld1 { v23.h }[2], [x26], #0x2\n" + "ld1 { v22.h }[2], [x25], #0x2\n" + "ld1 { v21.h }[2], [x24], #0x2\n" + "ld1 { v20.h }[2], [x23], #0x2\n" + "ld1 { v19.h }[2], [x22], #0x2\n" + "ld1 { v18.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v27.b }[6], [x28]\n" - "ld1 { v26.b }[6], [x27]\n" - "ld1 { v25.b }[6], [x26]\n" - "ld1 { v24.b }[6], [x25]\n" - "ld1 { v23.b }[6], [x24]\n" - "ld1 { v22.b }[6], [x23]\n" - "ld1 { v21.b }[6], [x22]\n" - "ld1 { v20.b }[6], [x21]\n" + "ld1 { v25.b }[6], [x28]\n" + "ld1 { v24.b }[6], [x27]\n" + "ld1 { v23.b }[6], [x26]\n" + "ld1 { v22.b }[6], [x25]\n" + "ld1 { v21.b }[6], [x24]\n" + "ld1 { v20.b }[6], [x23]\n" + "ld1 { v19.b }[6], [x22]\n" + "ld1 { v18.b }[6], [x21]\n" "b 11f\n" "8:" // odd_loads_1_4 "mov x20, #0x1\n" "tbz %x[width], #0, 11f\n" - "ld1 { v27.b }[4], [x28]\n" - "ld1 { v26.b }[4], [x27]\n" - "ld1 { v25.b }[4], [x26]\n" - "ld1 { v24.b }[4], [x25]\n" - "ld1 { v23.b }[4], [x24]\n" - "ld1 { v22.b }[4], [x23]\n" - "ld1 { v21.b }[4], [x22]\n" - "ld1 { v20.b }[4], [x21]\n" + "ld1 { v25.b }[4], [x28]\n" + "ld1 { v24.b }[4], [x27]\n" + "ld1 { v23.b }[4], [x26]\n" + "ld1 { v22.b }[4], [x25]\n" + "ld1 { v21.b }[4], [x24]\n" + "ld1 { v20.b }[4], [x23]\n" + "ld1 { v19.b }[4], [x22]\n" + "ld1 { v18.b }[4], [x21]\n" "b 11f\n" "9:" // odd_loads_2_0 "tbz %x[width], #1, 10f\n" - "ldr h27, [x28], #0x2\n" - "ldr h26, [x27], #0x2\n" + "ldr h25, [x28], #0x2\n" + "ldr h24, [x27], #0x2\n" "mov x20, #0x1\n" - "ldr h25, [x26], #0x2\n" - "ldr h24, [x25], #0x2\n" - "ldr h23, [x24], #0x2\n" - "ldr h22, [x23], #0x2\n" - "ldr h21, [x22], #0x2\n" - "ldr h20, [x21], #0x2\n" + "ldr h23, [x26], #0x2\n" + "ldr h22, [x25], #0x2\n" + "ldr h21, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h19, [x22], #0x2\n" + "ldr h18, [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v27.b }[2], [x28]\n" - "ld1 { v26.b }[2], [x27]\n" - "ld1 { v25.b }[2], [x26]\n" - "ld1 { v24.b }[2], [x25]\n" - "ld1 { v23.b }[2], [x24]\n" - "ld1 { v22.b }[2], [x23]\n" - "ld1 { v21.b }[2], [x22]\n" - "ld1 { v20.b }[2], [x21]\n" + "ld1 { v25.b }[2], [x28]\n" + "ld1 { v24.b }[2], [x27]\n" + "ld1 { v23.b }[2], [x26]\n" + "ld1 { v22.b }[2], [x25]\n" + "ld1 { v21.b }[2], [x24]\n" + "ld1 { v20.b }[2], [x23]\n" + "ld1 { v19.b }[2], [x22]\n" + "ld1 { v18.b }[2], [x21]\n" "b 11f\n" "10:" // odd_loads_1_0 - "ldr b27, [x28, #0x0]\n" - "ldr b26, [x27, #0x0]\n" + "ldr b25, [x28, #0x0]\n" + "ldr b24, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b25, [x26, #0x0]\n" - "ldr b24, [x25, #0x0]\n" - "ldr b23, [x24, #0x0]\n" - "ldr b22, [x23, #0x0]\n" - "ldr b21, [x22, #0x0]\n" - "ldr b20, [x21, #0x0]\n" + "ldr b23, [x26, #0x0]\n" + "ldr b22, [x25, #0x0]\n" + "ldr b21, [x24, #0x0]\n" + "ldr b20, [x23, #0x0]\n" + "ldr b19, [x22, #0x0]\n" + "ldr b18, [x21, #0x0]\n" "11:" // Odd load end "subs x20, x20, #0x1\n" - "zip1 v19.2d, v27.2d, v26.2d\n" - "zip1 v18.2d, v25.2d, v24.2d\n" - "zip1 v17.2d, v23.2d, v22.2d\n" - "zip1 v16.2d, v21.2d, v20.2d\n" - "str q19, [%x[out_ptr], #0x0]\n" - "str q18, [%x[out_ptr], #0x10]\n" + "zip1 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.2d, v21.2d, v20.2d\n" + "zip1 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "beq 12f\n" - "zip2 v19.2d, v27.2d, v26.2d\n" - "zip2 v18.2d, v25.2d, v24.2d\n" - "zip2 v17.2d, v23.2d, v22.2d\n" - "zip2 v16.2d, v21.2d, v20.2d\n" - "str q19, [%x[out_ptr], #0x0]\n" - "str q18, [%x[out_ptr], #0x10]\n" + "zip2 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v17.2d, v21.2d, v20.2d\n" + "zip2 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "12:" // Odds skip : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp index ff171984e7..8ed20a1e48 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp @@ -32,103 +32,103 @@ void interleave_block<1, 2, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "sub x28, %x[width], #0x1\n" - "mov x27, #0x0\n" "cntw x22, ALL, MUL #2\n" + "sub x28, %x[width], #0x1\n" "cntw x21, ALL, MUL #2\n" "sub x20, x22, #0x1\n" "whilelt p10.s, XZR, %x[height]\n" "add x28, x28, x21\n" - "ands x26, %x[width], x20\n" + "ands x27, %x[width], x20\n" "udiv x28, x28, x21\n" - "csel x26, x26, x22, NE\n" + "csel x27, x27, x22, NE\n" + "mov x26, #0x0\n" "and x25, x28, #0x1\n" "sub x28, x28, #0x1\n" - "add x26, x26, #0x1\n" + "add x27, x27, #0x1\n" "mov x20, %x[width]\n" "ptrue p0.b\n" "mov x24, %x[outptr_raw]\n" "mov x23, %x[row_offset]\n" "cntw x22\n" "lsr x28, x28, #0x1\n" - "lsr x26, x26, #0x1\n" + "lsr x27, x27, #0x1\n" "mov x12, #0x0\n" - ".inst 0x25b44771 // whilelt pn9.s, x27, x20, VLx2\n" + ".inst 0x25b44751 // whilelt pn9.s, x26, x20, VLx2\n" "mov x21, %x[in]\n" "1:" // Width loop: Preamble: Loop "ldr x20, [x21], #0x8\n" ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" - ".inst 0xa0174294 // ld1w { z20.s-z21.s }, pn8.s/Z, [x20, x23, LSL #2]\n" - ".inst 0xc160e294 // bfcvt z20.h, { z20.s-z21.s }\n" - ".inst 0xc0800280 // mova za0h.s[x12], p0/M, z20.s\n" + ".inst 0xa0174286 // ld1w { z6.s-z7.s }, pn8.s/Z, [x20, x23, LSL #2]\n" + ".inst 0xc160e0c6 // bfcvt z6.h, { z6.s-z7.s }\n" + ".inst 0xc08000c0 // mova za0h.s[x12], p0/M, z6.s\n" "add x12, x12, #0x1\n" "cmp x12, x22\n" "blt 1b\n" "incw x23, ALL, MUL #2\n" - "incw x27, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" "cbz x28, 5f\n" "2:" // Width loop "mov x20, %x[width]\n" "mov x12, #0x0\n" - ".inst 0x25b44771 // whilelt pn9.s, x27, x20, VLx2\n" + ".inst 0x25b44751 // whilelt pn9.s, x26, x20, VLx2\n" "mov x21, %x[in]\n" "3:" // Width loop: Odd: Loop "ldr x20, [x21], #0x8\n" ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" - ".inst 0xc0828018 // mova z24.s, p0/M, za0v.s[x12]\n" - ".inst 0xa0174294 // ld1w { z20.s-z21.s }, pn8.s/Z, [x20, x23, LSL #2]\n" - "st1w { z24.s }, p0, [x24]\n" - "addvl x24, x24, #1\n" - ".inst 0xc160e294 // bfcvt z20.h, { z20.s-z21.s }\n" - ".inst 0xc0800288 // mova za2h.s[x12], p0/M, z20.s\n" + ".inst 0xa017429e // ld1w { z30.s-z31.s }, pn8.s/Z, [x20, x23, LSL #2]\n" + ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" + ".inst 0xc08003c8 // mova za2h.s[x12], p0/M, z30.s\n" + ".inst 0xc082800f // mova z15.s, p0/M, za0v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x22\n" + "st1w { z15.s }, p0, [x24]\n" + "addvl x24, x24, #1\n" "blt 3b\n" - "incw x27, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" "mov x20, %x[width]\n" "incw x23, ALL, MUL #2\n" "mov x12, #0x0\n" - ".inst 0x25b44771 // whilelt pn9.s, x27, x20, VLx2\n" + ".inst 0x25b44751 // whilelt pn9.s, x26, x20, VLx2\n" "mov x21, %x[in]\n" "4:" // Width loop: Even: Loop "ldr x20, [x21], #0x8\n" ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" - ".inst 0xc082810c // mova z12.s, p0/M, za2v.s[x12]\n" - ".inst 0xa0174284 // ld1w { z4.s-z5.s }, pn8.s/Z, [x20, x23, LSL #2]\n" - "st1w { z12.s }, p0, [x24]\n" - "addvl x24, x24, #1\n" - ".inst 0xc160e084 // bfcvt z4.h, { z4.s-z5.s }\n" - ".inst 0xc0800080 // mova za0h.s[x12], p0/M, z4.s\n" + ".inst 0xa0174298 // ld1w { z24.s-z25.s }, pn8.s/Z, [x20, x23, LSL #2]\n" + ".inst 0xc160e318 // bfcvt z24.h, { z24.s-z25.s }\n" + ".inst 0xc0800300 // mova za0h.s[x12], p0/M, z24.s\n" + ".inst 0xc0828110 // mova z16.s, p0/M, za2v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x22\n" + "st1w { z16.s }, p0, [x24]\n" + "addvl x24, x24, #1\n" "blt 4b\n" "subs x28, x28, #0x1\n" "incw x23, ALL, MUL #2\n" - "incw x27, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" "bgt 2b\n" "5:" // Width loop: Tails "cbnz x25, 8f\n" "mov x20, %x[width]\n" "mov x12, #0x0\n" - ".inst 0x25b44771 // whilelt pn9.s, x27, x20, VLx2\n" + ".inst 0x25b44751 // whilelt pn9.s, x26, x20, VLx2\n" "mov x21, %x[in]\n" "6:" // Width loop: Tails: Even: Odd: Loop "ldr x20, [x21], #0x8\n" ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" - ".inst 0xc0828010 // mova z16.s, p0/M, za0v.s[x12]\n" ".inst 0xa017428e // ld1w { z14.s-z15.s }, pn8.s/Z, [x20, x23, LSL #2]\n" - "st1w { z16.s }, p0, [x24]\n" - "addvl x24, x24, #1\n" ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" ".inst 0xc08001c8 // mova za2h.s[x12], p0/M, z14.s\n" + ".inst 0xc0828010 // mova z16.s, p0/M, za0v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x22\n" + "st1w { z16.s }, p0, [x24]\n" + "addvl x24, x24, #1\n" "blt 6b\n" "mov x12, #0x0\n" "7:" // Width loop: Tails: Even: Even: Loop ".inst 0xc0828110 // mova z16.s, p0/M, za2v.s[x12]\n" "add x12, x12, #0x1\n" - "cmp x12, x26\n" + "cmp x12, x27\n" "st1w { z16.s }, p0, [x24]\n" "addvl x24, x24, #1\n" "blt 7b\n" @@ -138,7 +138,7 @@ void interleave_block<1, 2, VLType::SME, false>( "9:" // Width loop: Tails: Odd: Loop ".inst 0xc0828010 // mova z16.s, p0/M, za0v.s[x12]\n" "add x12, x12, #0x1\n" - "cmp x12, x26\n" + "cmp x12, x27\n" "st1w { z16.s }, p0, [x24]\n" "addvl x24, x24, #1\n" "blt 9b\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp index 6d3601d165..aaa01039cf 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp @@ -32,126 +32,126 @@ void interleave_block<2, 2, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "sub x10, %x[width], #0x1\n" - "mov x9, #0x0\n" "cntw x22, ALL, MUL #2\n" - "cntw x28\n" + "cntw x9\n" + "sub x28, %x[width], #0x1\n" "cntw x21, ALL, MUL #2\n" "sub x20, x22, #0x1\n" ".inst 0x25207815 // ptrue pn13.b\n" "whilelt p12.s, XZR, %x[height]\n" - "whilelt p11.s, x28, %x[height]\n" - "add x10, x10, x21\n" + "whilelt p11.s, x9, %x[height]\n" + "add x28, x28, x21\n" "ands x27, %x[width], x20\n" - "udiv x10, x10, x21\n" + "udiv x28, x28, x21\n" "csel x27, x27, x22, NE\n" - "and x26, x10, #0x1\n" - "sub x10, x10, #0x1\n" + "mov x26, #0x0\n" + "and x25, x28, #0x1\n" + "sub x28, x28, #0x1\n" "add x27, x27, #0x1\n" "mov x20, %x[width]\n" - "mov x25, %x[in]\n" + "mov x24, %x[in]\n" "ptrue p0.b\n" - "mov x24, %x[outptr_raw]\n" - "mov x23, %x[row_offset]\n" - "lsr x10, x10, #0x1\n" + "mov x23, %x[outptr_raw]\n" + "mov x22, %x[row_offset]\n" + "lsr x28, x28, #0x1\n" "lsr x27, x27, #0x1\n" "mov x12, #0x0\n" - ".inst 0x25b44532 // whilelt pn10.s, x9, x20, VLx2\n" - "add x22, x25, x28, LSL #3\n" + ".inst 0x25b44752 // whilelt pn10.s, x26, x20, VLx2\n" + "add x21, x24, x9, LSL #3\n" "1:" // Width loop: Preamble: Loop - "ldr x21, [x25], #0x8\n" + "ldr x20, [x24], #0x8\n" ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" - "ldr x20, [x22], #0x8\n" - ".inst 0xa01746b4 // ld1w { z20.s-z21.s }, pn9.s/Z, [x21, x23, LSL #2]\n" - ".inst 0xa017428c // ld1w { z12.s-z13.s }, pn8.s/Z, [x20, x23, LSL #2]\n" - ".inst 0xc160e294 // bfcvt z20.h, { z20.s-z21.s }\n" - ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" - ".inst 0xc0800280 // mova za0h.s[x12], p0/M, z20.s\n" - ".inst 0xc0800184 // mova za1h.s[x12], p0/M, z12.s\n" + ".inst 0xa0164698 // ld1w { z24.s-z25.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa0164296 // ld1w { z22.s-z23.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e318 // bfcvt z24.h, { z24.s-z25.s }\n" + ".inst 0xc160e2d6 // bfcvt z22.h, { z22.s-z23.s }\n" + ".inst 0xc0800300 // mova za0h.s[x12], p0/M, z24.s\n" + ".inst 0xc08002c4 // mova za1h.s[x12], p0/M, z22.s\n" "add x12, x12, #0x1\n" - "cmp x12, x28\n" + "cmp x12, x9\n" "blt 1b\n" - "incw x23, ALL, MUL #2\n" - "incw x9, ALL, MUL #2\n" - "cbz x10, 5f\n" + "incw x22, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" + "cbz x28, 5f\n" "2:" // Width loop "mov x20, %x[width]\n" - "mov x25, %x[in]\n" + "mov x24, %x[in]\n" "mov x12, #0x0\n" - ".inst 0x25b44532 // whilelt pn10.s, x9, x20, VLx2\n" - "add x22, x25, x28, LSL #3\n" + ".inst 0x25b44752 // whilelt pn10.s, x26, x20, VLx2\n" + "add x21, x24, x9, LSL #3\n" "3:" // Width loop: Odd: Loop - "ldr x21, [x25], #0x8\n" + "ldr x20, [x24], #0x8\n" ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" - ".inst 0xc0828007 // mova z7.s, p0/M, za0v.s[x12]\n" - "ldr x20, [x22], #0x8\n" - ".inst 0xc082808f // mova z15.s, p0/M, za1v.s[x12]\n" - ".inst 0xa01746b6 // ld1w { z22.s-z23.s }, pn9.s/Z, [x21, x23, LSL #2]\n" - ".inst 0xa017429a // ld1w { z26.s-z27.s }, pn8.s/Z, [x20, x23, LSL #2]\n" - ".inst 0xa1605707 // st1w { z7.s, z15.s }, pn13.b, [x24]\n" - "addvl x24, x24, #2\n" + ".inst 0xa0164696 // ld1w { z22.s-z23.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016428a // ld1w { z10.s-z11.s }, pn8.s/Z, [x20, x22, LSL #2]\n" ".inst 0xc160e2d6 // bfcvt z22.h, { z22.s-z23.s }\n" - ".inst 0xc160e35a // bfcvt z26.h, { z26.s-z27.s }\n" + ".inst 0xc160e14a // bfcvt z10.h, { z10.s-z11.s }\n" ".inst 0xc08002c8 // mova za2h.s[x12], p0/M, z22.s\n" - ".inst 0xc080034c // mova za3h.s[x12], p0/M, z26.s\n" + ".inst 0xc080014c // mova za3h.s[x12], p0/M, z10.s\n" + ".inst 0xc0828008 // mova z8.s, p0/M, za0v.s[x12]\n" + ".inst 0xc0828089 // mova z9.s, p0/M, za1v.s[x12]\n" "add x12, x12, #0x1\n" - "cmp x12, x28\n" + "cmp x12, x9\n" + ".inst 0xa06056e8 // st1w { z8.s-z9.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" "blt 3b\n" - "incw x9, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" "mov x20, %x[width]\n" - "mov x25, %x[in]\n" - "incw x23, ALL, MUL #2\n" + "mov x24, %x[in]\n" + "incw x22, ALL, MUL #2\n" "mov x12, #0x0\n" - ".inst 0x25b44532 // whilelt pn10.s, x9, x20, VLx2\n" - "add x22, x25, x28, LSL #3\n" + ".inst 0x25b44752 // whilelt pn10.s, x26, x20, VLx2\n" + "add x21, x24, x9, LSL #3\n" "4:" // Width loop: Even: Loop - "ldr x21, [x25], #0x8\n" + "ldr x20, [x24], #0x8\n" ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" - ".inst 0xc0828108 // mova z8.s, p0/M, za2v.s[x12]\n" - "ldr x20, [x22], #0x8\n" - ".inst 0xc0828189 // mova z9.s, p0/M, za3v.s[x12]\n" - ".inst 0xa01746ae // ld1w { z14.s-z15.s }, pn9.s/Z, [x21, x23, LSL #2]\n" - ".inst 0xa017428c // ld1w { z12.s-z13.s }, pn8.s/Z, [x20, x23, LSL #2]\n" - ".inst 0xa0605708 // st1w { z8.s-z9.s }, pn13.b, [x24]\n" - "addvl x24, x24, #2\n" - ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" - ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" - ".inst 0xc08001c0 // mova za0h.s[x12], p0/M, z14.s\n" - ".inst 0xc0800184 // mova za1h.s[x12], p0/M, z12.s\n" + ".inst 0xa016469a // ld1w { z26.s-z27.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016429e // ld1w { z30.s-z31.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e35a // bfcvt z26.h, { z26.s-z27.s }\n" + ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" + ".inst 0xc0800340 // mova za0h.s[x12], p0/M, z26.s\n" + ".inst 0xc08003c4 // mova za1h.s[x12], p0/M, z30.s\n" + ".inst 0xc0828106 // mova z6.s, p0/M, za2v.s[x12]\n" + ".inst 0xc082818e // mova z14.s, p0/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" - "cmp x12, x28\n" + "cmp x12, x9\n" + ".inst 0xa16056e6 // st1w { z6.s, z14.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" "blt 4b\n" - "subs x10, x10, #0x1\n" - "incw x23, ALL, MUL #2\n" - "incw x9, ALL, MUL #2\n" + "subs x28, x28, #0x1\n" + "incw x22, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" "bgt 2b\n" "5:" // Width loop: Tails - "cbnz x26, 8f\n" + "cbnz x25, 8f\n" "mov x20, %x[width]\n" - "mov x25, %x[in]\n" + "mov x24, %x[in]\n" "mov x12, #0x0\n" - ".inst 0x25b44532 // whilelt pn10.s, x9, x20, VLx2\n" - "add x22, x25, x28, LSL #3\n" + ".inst 0x25b44752 // whilelt pn10.s, x26, x20, VLx2\n" + "add x21, x24, x9, LSL #3\n" "6:" // Width loop: Tails: Even: Odd: Loop - "ldr x21, [x25], #0x8\n" + "ldr x20, [x24], #0x8\n" ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" - ".inst 0xc0828003 // mova z3.s, p0/M, za0v.s[x12]\n" - "ldr x20, [x22], #0x8\n" - ".inst 0xc082808b // mova z11.s, p0/M, za1v.s[x12]\n" - ".inst 0xa01746ac // ld1w { z12.s-z13.s }, pn9.s/Z, [x21, x23, LSL #2]\n" - ".inst 0xa017428e // ld1w { z14.s-z15.s }, pn8.s/Z, [x20, x23, LSL #2]\n" - ".inst 0xa1605703 // st1w { z3.s, z11.s }, pn13.b, [x24]\n" - "addvl x24, x24, #2\n" + ".inst 0xa016468c // ld1w { z12.s-z13.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016428e // ld1w { z14.s-z15.s }, pn8.s/Z, [x20, x22, LSL #2]\n" ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" ".inst 0xc0800188 // mova za2h.s[x12], p0/M, z12.s\n" ".inst 0xc08001cc // mova za3h.s[x12], p0/M, z14.s\n" + ".inst 0xc0828007 // mova z7.s, p0/M, za0v.s[x12]\n" + ".inst 0xc082808f // mova z15.s, p0/M, za1v.s[x12]\n" "add x12, x12, #0x1\n" - "cmp x12, x28\n" + "cmp x12, x9\n" + ".inst 0xa16056e7 // st1w { z7.s, z15.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" "blt 6b\n" "mov x12, #0x0\n" "7:" // Width loop: Tails: Even: Even: Loop @@ -159,8 +159,8 @@ void interleave_block<2, 2, VLType::SME, false>( ".inst 0xc082818f // mova z15.s, p0/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x27\n" - ".inst 0xa060570e // st1w { z14.s-z15.s }, pn13.b, [x24]\n" - "addvl x24, x24, #2\n" + ".inst 0xa06056ee // st1w { z14.s-z15.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" "blt 7b\n" "b 10f\n" "8:" // Width loop: Tails: Odd @@ -170,15 +170,15 @@ void interleave_block<2, 2, VLType::SME, false>( ".inst 0xc0828095 // mova z21.s, p0/M, za1v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x27\n" - ".inst 0xa0605714 // st1w { z20.s-z21.s }, pn13.b, [x24]\n" - "addvl x24, x24, #2\n" + ".inst 0xa06056f4 // st1w { z20.s-z21.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" "blt 9b\n" "10:" // End - "mov %x[outptr_raw], x24\n" + "mov %x[outptr_raw], x23\n" ".inst 0xd503467f // SMSTOP\n" : [outptr_raw] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp index a8187f78e8..83a7f62693 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp @@ -32,63 +32,63 @@ void interleave_block<4, 2, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "sub x14, %x[width], #0x1\n" - "mov x13, %x[in]\n" "cntw x23, ALL, MUL #2\n" - "cntw x11\n" + "cntw x10\n" "cntw x22, ALL, MUL #2\n" "cntw x20, ALL, MUL #3\n" "sub x21, x23, #0x1\n" ".inst 0x25207817 // ptrue pn15.b\n" - "whilelt p2.s, XZR, %x[height]\n" - "whilelt p1.s, x11, %x[height]\n" - "whilelt p14.s, x22, %x[height]\n" - "whilelt p13.s, x20, %x[height]\n" + "whilelt p1.s, XZR, %x[height]\n" + "whilelt p14.s, x10, %x[height]\n" + "whilelt p13.s, x22, %x[height]\n" + "whilelt p12.s, x20, %x[height]\n" + "sub x9, %x[width], #0x1\n" "cntw x20, ALL, MUL #2\n" - "ands x10, %x[width], x21\n" - "add x14, x14, x20\n" - "csel x10, x10, x23, NE\n" - "add x9, x13, x11, LSL #3\n" - "mov x28, #0x0\n" - "udiv x14, x14, x20\n" - "add x10, x10, #0x1\n" + "ands x28, %x[width], x21\n" + "mov x27, %x[in]\n" + "add x9, x9, x20\n" + "csel x28, x28, x23, NE\n" + "add x26, x27, x10, LSL #3\n" + "mov x25, #0x0\n" + "udiv x9, x9, x20\n" + "add x28, x28, #0x1\n" "mov x20, %x[width]\n" - "add x27, x9, x11, LSL #3\n" + "add x24, x26, x10, LSL #3\n" "ptrue p0.b\n" - "mov x26, %x[outptr_raw]\n" - "mov x25, %x[row_offset]\n" - "sub x14, x14, #0x1\n" - "lsr x10, x10, #0x1\n" + "mov x23, %x[outptr_raw]\n" + "mov x22, %x[row_offset]\n" + "sub x9, x9, #0x1\n" + "lsr x28, x28, #0x1\n" "mov x12, #0x0\n" - ".inst 0x25b44794 // whilelt pn12.s, x28, x20, VLx2\n" - "add x24, x27, x11, LSL #3\n" + ".inst 0x25b44733 // whilelt pn11.s, x25, x20, VLx2\n" + "add x21, x24, x10, LSL #3\n" "1:" // Width loop: Preamble: Loop - "ldr x23, [x13], #0x8\n" - ".inst 0x2530704b // psel p11.s, p12.s/Z, p2.s[w12]\n" - ".inst 0x2530702a // psel p10.s, p12.s/Z, p1.s[w12]\n" - "ldr x22, [x9], #0x8\n" - ".inst 0x253071c9 // psel p9.s, p12.s/Z, p14.s[w12]\n" - ".inst 0x253071a8 // psel p8.s, p12.s/Z, p13.s[w12]\n" - "ldr x21, [x27], #0x8\n" + "ldr x20, [x27], #0x8\n" + ".inst 0x25306c28 // psel p8.s, p11.s/Z, p1.s[w12]\n" + ".inst 0x25306dca // psel p10.s, p11.s/Z, p14.s[w12]\n" + ".inst 0xa0164298 // ld1w { z24.s-z25.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0x25306da9 // psel p9.s, p11.s/Z, p13.s[w12]\n" + ".inst 0x25306d88 // psel p8.s, p11.s/Z, p12.s[w12]\n" + ".inst 0xa0164a82 // ld1w { z2.s-z3.s }, pn10.s/Z, [x20, x22, LSL #2]\n" "ldr x20, [x24], #0x8\n" - ".inst 0xa0194eea // ld1w { z10.s-z11.s }, pn11.s/Z, [x23, x25, LSL #2]\n" - ".inst 0xa0194ada // ld1w { z26.s-z27.s }, pn10.s/Z, [x22, x25, LSL #2]\n" - ".inst 0xa01946be // ld1w { z30.s-z31.s }, pn9.s/Z, [x21, x25, LSL #2]\n" - ".inst 0xa019428c // ld1w { z12.s-z13.s }, pn8.s/Z, [x20, x25, LSL #2]\n" + ".inst 0xa016468a // ld1w { z10.s-z11.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e318 // bfcvt z24.h, { z24.s-z25.s }\n" + ".inst 0xc160e042 // bfcvt z2.h, { z2.s-z3.s }\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016428c // ld1w { z12.s-z13.s }, pn8.s/Z, [x20, x22, LSL #2]\n" ".inst 0xc160e14a // bfcvt z10.h, { z10.s-z11.s }\n" - ".inst 0xc160e35a // bfcvt z26.h, { z26.s-z27.s }\n" - ".inst 0xc0800140 // mova za0h.s[x12], p0/M, z10.s\n" - ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" - ".inst 0xc0800344 // mova za1h.s[x12], p0/M, z26.s\n" ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" - ".inst 0xc08003c8 // mova za2h.s[x12], p0/M, z30.s\n" + ".inst 0xc0800300 // mova za0h.s[x12], p0/M, z24.s\n" + ".inst 0xc0800044 // mova za1h.s[x12], p0/M, z2.s\n" + ".inst 0xc0800148 // mova za2h.s[x12], p0/M, z10.s\n" ".inst 0xc080018c // mova za3h.s[x12], p0/M, z12.s\n" "add x12, x12, #0x1\n" - "cmp x12, x11\n" + "cmp x12, x10\n" "blt 1b\n" + "incw x22, ALL, MUL #2\n" "incw x25, ALL, MUL #2\n" - "incw x28, ALL, MUL #2\n" - "cbz x14, 5f\n" + "cbz x9, 5f\n" "2:" // Width loop "mov x12, #0x0\n" "3:" // Width loop: Store: Loop @@ -97,44 +97,44 @@ void interleave_block<4, 2, VLType::SME, false>( ".inst 0xc0828119 // mova z25.s, p0/M, za2v.s[x12]\n" ".inst 0xc082819d // mova z29.s, p0/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" - "cmp x12, x11\n" - ".inst 0xa160df51 // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x26]\n" - "addvl x26, x26, #4\n" + "cmp x12, x10\n" + ".inst 0xa160def1 // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x23]\n" + "addvl x23, x23, #4\n" "blt 3b\n" - "mov x13, %x[in]\n" + "mov x27, %x[in]\n" + "add x26, x27, x10, LSL #3\n" "mov x20, %x[width]\n" - "add x9, x13, x11, LSL #3\n" + "add x24, x26, x10, LSL #3\n" "mov x12, #0x0\n" - "add x27, x9, x11, LSL #3\n" - ".inst 0x25b44794 // whilelt pn12.s, x28, x20, VLx2\n" - "add x24, x27, x11, LSL #3\n" + ".inst 0x25b44733 // whilelt pn11.s, x25, x20, VLx2\n" + "add x21, x24, x10, LSL #3\n" "4:" // Width loop: Load: Loop - "ldr x23, [x13], #0x8\n" - ".inst 0x2530704b // psel p11.s, p12.s/Z, p2.s[w12]\n" - ".inst 0x2530702a // psel p10.s, p12.s/Z, p1.s[w12]\n" - "ldr x22, [x9], #0x8\n" - ".inst 0x253071c9 // psel p9.s, p12.s/Z, p14.s[w12]\n" - ".inst 0x253071a8 // psel p8.s, p12.s/Z, p13.s[w12]\n" - "ldr x21, [x27], #0x8\n" + "ldr x20, [x27], #0x8\n" + ".inst 0x25306c28 // psel p8.s, p11.s/Z, p1.s[w12]\n" + ".inst 0x25306dca // psel p10.s, p11.s/Z, p14.s[w12]\n" + ".inst 0xa016428c // ld1w { z12.s-z13.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0x25306da9 // psel p9.s, p11.s/Z, p13.s[w12]\n" + ".inst 0x25306d88 // psel p8.s, p11.s/Z, p12.s[w12]\n" + ".inst 0xa0164a8e // ld1w { z14.s-z15.s }, pn10.s/Z, [x20, x22, LSL #2]\n" "ldr x20, [x24], #0x8\n" - ".inst 0xa0194eec // ld1w { z12.s-z13.s }, pn11.s/Z, [x23, x25, LSL #2]\n" - ".inst 0xa0194ace // ld1w { z14.s-z15.s }, pn10.s/Z, [x22, x25, LSL #2]\n" - ".inst 0xa01946b2 // ld1w { z18.s-z19.s }, pn9.s/Z, [x21, x25, LSL #2]\n" - ".inst 0xa019429e // ld1w { z30.s-z31.s }, pn8.s/Z, [x20, x25, LSL #2]\n" + ".inst 0xa0164692 // ld1w { z18.s-z19.s }, pn9.s/Z, [x20, x22, LSL #2]\n" ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" - ".inst 0xc0800180 // mova za0h.s[x12], p0/M, z12.s\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016429e // ld1w { z30.s-z31.s }, pn8.s/Z, [x20, x22, LSL #2]\n" ".inst 0xc160e252 // bfcvt z18.h, { z18.s-z19.s }\n" - ".inst 0xc08001c4 // mova za1h.s[x12], p0/M, z14.s\n" ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" + ".inst 0xc0800180 // mova za0h.s[x12], p0/M, z12.s\n" + ".inst 0xc08001c4 // mova za1h.s[x12], p0/M, z14.s\n" ".inst 0xc0800248 // mova za2h.s[x12], p0/M, z18.s\n" ".inst 0xc08003cc // mova za3h.s[x12], p0/M, z30.s\n" "add x12, x12, #0x1\n" - "cmp x12, x11\n" + "cmp x12, x10\n" "blt 4b\n" - "subs x14, x14, #0x1\n" + "subs x9, x9, #0x1\n" + "incw x22, ALL, MUL #2\n" "incw x25, ALL, MUL #2\n" - "incw x28, ALL, MUL #2\n" "bgt 2b\n" "5:" // Width loop: Tails "mov x12, #0x0\n" @@ -144,16 +144,16 @@ void interleave_block<4, 2, VLType::SME, false>( ".inst 0xc0828119 // mova z25.s, p0/M, za2v.s[x12]\n" ".inst 0xc082819d // mova z29.s, p0/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" - "cmp x12, x10\n" - ".inst 0xa160df51 // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x26]\n" - "addvl x26, x26, #4\n" + "cmp x12, x28\n" + ".inst 0xa160def1 // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x23]\n" + "addvl x23, x23, #4\n" "blt 6b\n" "7:" // End - "mov %x[outptr_raw], x26\n" + "mov %x[outptr_raw], x23\n" ".inst 0xd503467f // SMSTOP\n" : [outptr_raw] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp index bc9f68ed72..45f660fec1 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp @@ -33,25 +33,25 @@ void interleave_block<1, 1, VLType::SME, false>( __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "mov x21, %x[width]\n" - "mov x20, %x[width]\n" "inch x21\n" "cnth x11\n" "sub x21, x21, #0x1\n" - "sub x10, x11, #0x1\n" "udiv x21, x21, x11\n" // n_passes = ceildiv(width, VL<T>) - "ands x10, x20, x10\n" + "mov x20, %x[width]\n" + "sub x10, x11, #0x1\n" "sub x9, x21, #0x1\n" + "ands x10, x20, x10\n" "sub x28, x11, #0x2\n" "lsl x20, %x[height], #0x1\n" // height * 2 "mov x27, #0x0\n" "mov x26, %x[in]\n" "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x25, [x26, #0x0]\n" + "and x24, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) "csel x10, x10, x11, NE\n" - "ldr x24, [x26, #0x0]\n" + "ldr x23, [x26, #0x8]\n" "ptrue p11.h\n" "whilelt p10.h, XZR, x20\n" - "ldr x23, [x26, #0x8]\n" "mov x22, %x[row_offset]\n" "mov x21, %x[out]\n" "whilelt p9.h, x27, %x[width]\n" @@ -60,119 +60,119 @@ void interleave_block<1, 1, VLType::SME, false>( "mov x12, #0x0\n" "cbz x28, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25286141 // psel p1.h, p8.h/Z, p10.h[w12]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0xe0560700 // ld1h { za0h.h[x12] }, p1/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" + "ldr x25, [x26, #0x0]\n" ".inst 0xe05602e1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x23, x22, LSL #1]\n" "add x12, x12, #0x2\n" + "cmp x12, x28\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "cmp x12, x28\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25286141 // psel p1.h, p8.h/Z, p10.h[w12]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" "mov x26, %x[in]\n" - "inch x27\n" - ".inst 0xe0560700 // ld1h { za0h.h[x12] }, p1/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" ".inst 0xe05602e1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x23, x22, LSL #1]\n" + "ldr x25, [x26, #0x0]\n" + "inch x22\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "inch x22\n" + "inch x27\n" "cbz x9, 8f\n" "mov x20, x9\n" "3:" // K loop: Main loop "whilelt p8.h, x27, %x[width]\n" - "mov x13, #0x0\n" + "mov x12, #0x0\n" "cbz x28, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25296143 // psel p3.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25396142 // psel p2.h, p8.h/Z, p10.h[w13, #1]\n" - ".inst 0x25296d21 // psel p1.h, p11.h/Z, p9.h[w13]\n" - ".inst 0x25396d20 // psel p0.h, p11.h/Z, p9.h[w13, #1]\n" - ".inst 0xe0562f08 // ld1h { za1h.h[x13] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0562ae9 // ld1h { za1h.h[x13, #1] }, p2/Z, [x23, x22, LSL #1]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" "ldr x23, [x26, #0x8]\n" - ".inst 0xe07fa6a0 // st1h { za0v.h[x13] }, p1/Z, [x21, XZR, LSL #1]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06b82a1 // st1h { za0v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" "add x26, x26, #0x10\n" - ".inst 0xe06ba2a1 // st1h { za0v.h[x13, #1] }, p0/Z, [x21, x11, LSL #1]\n" - "add x13, x13, #0x2\n" "addvl x21, x21, #2\n" - "cmp x13, x28\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25296143 // psel p3.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25396142 // psel p2.h, p8.h/Z, p10.h[w13, #1]\n" - ".inst 0x25296d21 // psel p1.h, p11.h/Z, p9.h[w13]\n" - ".inst 0x25396d20 // psel p0.h, p11.h/Z, p9.h[w13, #1]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" "whilelt p9.h, x27, %x[width]\n" - ".inst 0xe0562f08 // ld1h { za1h.h[x13] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" "inch x27\n" - "mov x12, #0x0\n" - ".inst 0xe0562ae9 // ld1h { za1h.h[x13, #1] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x26, #0x8]\n" - ".inst 0xe07fa6a0 // st1h { za0v.h[x13] }, p1/Z, [x21, XZR, LSL #1]\n" "add x26, x26, #0x10\n" - ".inst 0xe06ba2a1 // st1h { za0v.h[x13, #1] }, p0/Z, [x21, x11, LSL #1]\n" + ".inst 0xe06b82a1 // st1h { za0v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" "addvl x21, x21, #2\n" "inch x22\n" "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" "cbz x28, 7f\n" "6:" // K loop: Main loop: Second: Loop - ".inst 0x25286143 // psel p3.h, p8.h/Z, p10.h[w12]\n" - ".inst 0x25386142 // psel p2.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" - ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" - ".inst 0xe0560f00 // ld1h { za0h.h[x12] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0560ae1 // ld1h { za0h.h[x12, #1] }, p2/Z, [x23, x22, LSL #1]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" "ldr x23, [x26, #0x8]\n" - ".inst 0xe07f86a8 // st1h { za1v.h[x12] }, p1/Z, [x21, XZR, LSL #1]\n" - "add x26, x26, #0x10\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" ".inst 0xe06b82a9 // st1h { za1v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" "add x12, x12, #0x2\n" - "addvl x21, x21, #2\n" "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail - ".inst 0x25286143 // psel p3.h, p8.h/Z, p10.h[w12]\n" - ".inst 0x25386142 // psel p2.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" - ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" "whilelt p9.h, x27, %x[width]\n" - ".inst 0xe0560f00 // ld1h { za0h.h[x12] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" "subs x20, x20, #0x1\n" - "inch x27\n" - ".inst 0xe0560ae1 // ld1h { za0h.h[x12, #1] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x26, #0x8]\n" - ".inst 0xe07f86a8 // st1h { za1v.h[x12] }, p1/Z, [x21, XZR, LSL #1]\n" "add x26, x26, #0x10\n" ".inst 0xe06b82a9 // st1h { za1v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" "addvl x21, x21, #2\n" + "inch x27\n" "inch x22\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x25, 11f\n" + "cbnz x24, 11f\n" "mov x26, %x[in]\n" "whilelt p8.h, x27, %x[width]\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" - ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" - ".inst 0xe07f86a0 // st1h { za0v.h[x12] }, p1/Z, [x21, XZR, LSL #1]\n" - "addvl x21, x21, #1\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" "ldr x20, [x26, #0x0]\n" - "add x26, x26, #0x8\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" ".inst 0xe0560288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x22, LSL #1]\n" "add x12, x12, #0x1\n" "cmp x12, x11\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" "blt 9b\n" "whilelt p9.h, x27, %x[width]\n" "whilelt p8.h, x27, %x[width]\n" @@ -181,8 +181,8 @@ void interleave_block<1, 1, VLType::SME, false>( ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x10\n" + "addvl x21, x21, #1\n" "blt 10b\n" "whilelt p8.h, x27, %x[width]\n" "b 13f\n" @@ -192,15 +192,15 @@ void interleave_block<1, 1, VLType::SME, false>( ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x10\n" + "addvl x21, x21, #1\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x21\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp index 793bc80524..ce7192afe6 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp @@ -32,28 +32,28 @@ void interleave_block<1, 2, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x22, %x[width]\n" + "cnth x22\n" "mov x21, %x[width]\n" - "cnth x20\n" - "inch x22\n" - "sub x11, x20, #0x1\n" - "sub x22, x22, #0x1\n" - "ands x11, x21, x11\n" + "inch x21\n" + "mov x20, %x[width]\n" + "sub x11, x22, #0x1\n" + "sub x21, x21, #0x1\n" + "ands x11, x20, x11\n" "cntw x10\n" - "udiv x22, x22, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x11, x11, x20, NE\n" - "sub x9, x22, #0x1\n" + "udiv x21, x21, x22\n" // n_passes = ceildiv(width, VL<T>) + "csel x11, x11, x22, NE\n" + "sub x9, x21, #0x1\n" "add x11, x11, #0x1\n" "sub x28, x10, #0x2\n" "lsl x20, %x[height], #0x1\n" // height * 2 "mov x27, #0x0\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x25, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) - "ldr x24, [x26, #0x0]\n" + "and x24, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x23, [x26, #0x8]\n" "lsr x11, x11, #0x1\n" "ptrue p11.s\n" - "ldr x23, [x26, #0x8]\n" "whilelt p10.h, XZR, x20\n" "mov x22, %x[row_offset]\n" "mov x21, %x[out]\n" @@ -63,124 +63,124 @@ void interleave_block<1, 2, VLType::SME, false>( "mov x12, #0x0\n" "cbz x28, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25286141 // psel p1.h, p8.h/Z, p10.h[w12]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" - ".inst 0xe0560700 // ld1h { za0h.h[x12] }, p1/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" + "ldr x25, [x26, #0x0]\n" ".inst 0xe05602e2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" "add x12, x12, #0x4\n" + "cmp x12, x28, LSL #1\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "cmp x12, x28, LSL #1\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25286141 // psel p1.h, p8.h/Z, p10.h[w12]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" "mov x26, %x[in]\n" - "inch x27\n" - ".inst 0xe0560700 // ld1h { za0h.h[x12] }, p1/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" ".inst 0xe05602e2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" + "ldr x25, [x26, #0x0]\n" + "inch x22\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "inch x22\n" + "inch x27\n" "cbz x9, 8f\n" "mov x20, x9\n" "3:" // K loop: Main loop "whilelt p8.h, x27, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - "mov x14, #0x0\n" "cbz x28, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25386143 // psel p3.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0x25786142 // psel p2.h, p8.h/Z, p10.h[w12, #3]\n" - ".inst 0x252a6d21 // psel p1.h, p11.h/Z, p9.h[w14]\n" - ".inst 0x253a6d20 // psel p0.h, p11.h/Z, p9.h[w14, #1]\n" - ".inst 0xe0560f01 // ld1h { za0h.h[x12, #1] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0560ae3 // ld1h { za0h.h[x12, #3] }, p2/Z, [x23, x22, LSL #1]\n" + ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" + ".inst 0xe0562321 // ld1h { za0h.h[x13, #1] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25796141 // psel p1.h, p8.h/Z, p10.h[w13, #3]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe05626e3 // ld1h { za0h.h[x13, #3] }, p1/Z, [x23, x22, LSL #1]\n" "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0aa82a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" "add x26, x26, #0x10\n" - "add x12, x12, #0x4\n" - ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" - ".inst 0xe0aac2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x10, LSL #2]\n" - "add x14, x14, #0x2\n" "addvl x21, x21, #2\n" - "cmp x14, x28\n" + "add x13, x13, #0x4\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25386143 // psel p3.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0x25786142 // psel p2.h, p8.h/Z, p10.h[w12, #3]\n" - ".inst 0x252a6d21 // psel p1.h, p11.h/Z, p9.h[w14]\n" - ".inst 0x253a6d20 // psel p0.h, p11.h/Z, p9.h[w14, #1]\n" + ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" + ".inst 0xe0562321 // ld1h { za0h.h[x13, #1] }, p0/Z, [x25, x22, LSL #1]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25796141 // psel p1.h, p8.h/Z, p10.h[w13, #3]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe05626e3 // ld1h { za0h.h[x13, #3] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "whilelt p9.h, x27, %x[width]\n" - ".inst 0xe0560f01 // ld1h { za0h.h[x12, #1] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" "inch x27\n" - "mov x13, #0x0\n" - ".inst 0xe0560ae3 // ld1h { za0h.h[x12, #3] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" + ".inst 0xe0aa82a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "addvl x21, x21, #2\n" "inch x22\n" - ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" "whilelt p8.h, x27, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - ".inst 0xe0aac2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x10, LSL #2]\n" - "addvl x21, x21, #2\n" "cbz x28, 7f\n" "6:" // K loop: Main loop: Second: Loop - ".inst 0x25296143 // psel p3.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25696142 // psel p2.h, p8.h/Z, p10.h[w13, #2]\n" - ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" - ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" - ".inst 0xe0562f00 // ld1h { za0h.h[x13] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0562ae2 // ld1h { za0h.h[x13, #2] }, p2/Z, [x23, x22, LSL #1]\n" + ".inst 0x25296140 // psel p0.h, p8.h/Z, p10.h[w13]\n" + ".inst 0xe0562320 // ld1h { za0h.h[x13] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25696141 // psel p1.h, p8.h/Z, p10.h[w13, #2]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe05626e2 // ld1h { za0h.h[x13, #2] }, p1/Z, [x23, x22, LSL #1]\n" "ldr x23, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - "add x13, x13, #0x4\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" "add x12, x12, #0x2\n" - "addvl x21, x21, #2\n" "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x4\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail - ".inst 0x25296143 // psel p3.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25696142 // psel p2.h, p8.h/Z, p10.h[w13, #2]\n" - ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" - ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0x25296140 // psel p0.h, p8.h/Z, p10.h[w13]\n" + ".inst 0xe0562320 // ld1h { za0h.h[x13] }, p0/Z, [x25, x22, LSL #1]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25696141 // psel p1.h, p8.h/Z, p10.h[w13, #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe05626e2 // ld1h { za0h.h[x13, #2] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "whilelt p9.h, x27, %x[width]\n" - ".inst 0xe0562f00 // ld1h { za0h.h[x13] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" "subs x20, x20, #0x1\n" - "inch x27\n" - ".inst 0xe0562ae2 // ld1h { za0h.h[x13, #2] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "inch x22\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" "addvl x21, x21, #2\n" + "inch x27\n" + "inch x22\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x25, 11f\n" + "cbnz x24, 11f\n" "mov x26, %x[in]\n" "whilelt p8.h, x27, %x[width]\n" "mov x13, #0x0\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" - ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" - "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "ldr x20, [x26, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" "cmp x12, x10\n" - "add x26, x26, #0x8\n" ".inst 0xe0562281 // ld1h { za0h.h[x13, #1] }, p0/Z, [x20, x22, LSL #1]\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" "add x13, x13, #0x2\n" "blt 9b\n" "whilelt p9.h, x27, %x[width]\n" @@ -189,11 +189,11 @@ void interleave_block<1, 2, VLType::SME, false>( "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - "add x20, x20, #0x2\n" ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x11\n" + "addvl x21, x21, #1\n" + "add x20, x20, #0x2\n" "blt 10b\n" "whilelt p8.h, x27, %x[width]\n" "b 13f\n" @@ -203,15 +203,15 @@ void interleave_block<1, 2, VLType::SME, false>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x11\n" + "addvl x21, x21, #1\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x21\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_fp16_fp16.hpp index 0e0e4e462c..982cfa6d40 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_fp16_fp16.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME) +#if defined(__ARM_FEATURE_SVE) template <> void interleave_block<1, 2, VLType::SME, false>( @@ -63,11 +63,11 @@ void interleave_block<1, 2, VLType::SME, false>( "mov x12, #0x0\n" "cbz x28, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25286141 // psel p1.h, p8.h/Z, p10.h[w12]\n" - ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" - ".inst 0xe0560700 // ld1h { za0h.h[x12] }, p1/Z, [x24, x22, LSL #1]\n" + ".inst 0x25286143 // psel p3.h, p8.h/Z, p10.h[w12]\n" + ".inst 0x25686142 // psel p2.h, p8.h/Z, p10.h[w12, #2]\n" + ".inst 0xe0560f00 // ld1h { za0h.h[x12] }, p3/Z, [x24, x22, LSL #1]\n" "ldr x24, [x26, #0x0]\n" - ".inst 0xe05602e2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" + ".inst 0xe0560ae2 // ld1h { za0h.h[x12, #2] }, p2/Z, [x23, x22, LSL #1]\n" "add x12, x12, #0x4\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" @@ -88,20 +88,20 @@ void interleave_block<1, 2, VLType::SME, false>( "mov x20, x9\n" "3:" // K loop: Main loop "whilelt p8.h, x27, %x[width]\n" - "mov x12, #0x0\n" + "mov x15, #0x0\n" "mov x14, #0x0\n" "cbz x28, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25386143 // psel p3.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0x25786142 // psel p2.h, p8.h/Z, p10.h[w12, #3]\n" + ".inst 0x253b6143 // psel p3.h, p8.h/Z, p10.h[w15, #1]\n" + ".inst 0x257b6142 // psel p2.h, p8.h/Z, p10.h[w15, #3]\n" ".inst 0x252a6d21 // psel p1.h, p11.h/Z, p9.h[w14]\n" ".inst 0x253a6d20 // psel p0.h, p11.h/Z, p9.h[w14, #1]\n" - ".inst 0xe0560f01 // ld1h { za0h.h[x12, #1] }, p3/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0566f01 // ld1h { za0h.h[x15, #1] }, p3/Z, [x24, x22, LSL #1]\n" "ldr x24, [x26, #0x0]\n" - ".inst 0xe0560ae3 // ld1h { za0h.h[x12, #3] }, p2/Z, [x23, x22, LSL #1]\n" + ".inst 0xe0566ae3 // ld1h { za0h.h[x15, #3] }, p2/Z, [x23, x22, LSL #1]\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "add x12, x12, #0x4\n" + "add x15, x15, #0x4\n" ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" ".inst 0xe0aac2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x10, LSL #2]\n" "add x14, x14, #0x2\n" @@ -109,23 +109,23 @@ void interleave_block<1, 2, VLType::SME, false>( "cmp x14, x28\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25386143 // psel p3.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0x25786142 // psel p2.h, p8.h/Z, p10.h[w12, #3]\n" + ".inst 0x253b6143 // psel p3.h, p8.h/Z, p10.h[w15, #1]\n" + ".inst 0x257b6142 // psel p2.h, p8.h/Z, p10.h[w15, #3]\n" ".inst 0x252a6d21 // psel p1.h, p11.h/Z, p9.h[w14]\n" ".inst 0x253a6d20 // psel p0.h, p11.h/Z, p9.h[w14, #1]\n" "mov x26, %x[in]\n" "whilelt p9.h, x27, %x[width]\n" - ".inst 0xe0560f01 // ld1h { za0h.h[x12, #1] }, p3/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0566f01 // ld1h { za0h.h[x15, #1] }, p3/Z, [x24, x22, LSL #1]\n" "ldr x24, [x26, #0x0]\n" "inch x27\n" "mov x13, #0x0\n" - ".inst 0xe0560ae3 // ld1h { za0h.h[x12, #3] }, p2/Z, [x23, x22, LSL #1]\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + ".inst 0xe0566ae3 // ld1h { za0h.h[x15, #3] }, p2/Z, [x23, x22, LSL #1]\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" "inch x22\n" ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" - "whilelt p8.h, x27, %x[width]\n" - "mov x12, #0x0\n" ".inst 0xe0aac2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x10, LSL #2]\n" "addvl x21, x21, #2\n" "cbz x28, 7f\n" @@ -172,15 +172,15 @@ void interleave_block<1, 2, VLType::SME, false>( "mov x13, #0x0\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" - ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25396143 // psel p3.h, p8.h/Z, p10.h[w13, #1]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x24, [x26, #0x0]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" - "ldr x20, [x26, #0x0]\n" - "cmp x12, x10\n" "add x26, x26, #0x8\n" - ".inst 0xe0562281 // ld1h { za0h.h[x13, #1] }, p0/Z, [x20, x22, LSL #1]\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + ".inst 0xe0562f01 // ld1h { za0h.h[x13, #1] }, p3/Z, [x24, x22, LSL #1]\n" "add x13, x13, #0x2\n" "blt 9b\n" "whilelt p9.h, x27, %x[width]\n" @@ -195,7 +195,7 @@ void interleave_block<1, 2, VLType::SME, false>( "addvl x21, x21, #1\n" "cmp x12, x11\n" "blt 10b\n" - "whilelt p8.h, x27, %x[width]\n" + "whilelt p9.h, x27, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -203,16 +203,16 @@ void interleave_block<1, 2, VLType::SME, false>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x11\n" + "addvl x21, x21, #1\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x21\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } -#endif // defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp index 558a5d7637..79cd668a84 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp @@ -32,16 +32,16 @@ void interleave_block<1, 4, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" "mov x23, %x[width]\n" - "mov x21, %x[width]\n" - "cntb x20\n" "incb x23\n" - "sub x10, x20, #0x1\n" + "mov x20, %x[width]\n" + "sub x10, x21, #0x1\n" "cntw x9\n" "sub x23, x23, #0x1\n" - "ands x10, x21, x10\n" - "udiv x23, x23, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x10, x10, x20, NE\n" + "ands x10, x20, x10\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x10, x10, x21, NE\n" "lsl x22, %x[height], #0x1\n" // height * 2 "lsl x21, x9, #0x1\n" "sub x20, x23, #0x1\n" @@ -52,12 +52,12 @@ void interleave_block<1, 4, VLType::SME, false>( "mov x27, #0x0\n" "mov x26, %x[in]\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x25, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x25, [x26, #0x0]\n" + "and x24, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) "lsr x10, x10, #0x2\n" - "ldr x24, [x26, #0x0]\n" + "ldr x23, [x26, #0x8]\n" "ptrue p11.s\n" "zip1 p10.b, p9.b, p8.b\n" - "ldr x23, [x26, #0x8]\n" "mov x22, %x[row_offset]\n" "mov x21, %x[out]\n" "whilelt p9.b, x27, %x[width]\n" @@ -66,124 +66,124 @@ void interleave_block<1, 4, VLType::SME, false>( "mov x12, #0x0\n" "cbz x28, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25246141 // psel p1.b, p8.b/Z, p10.b[w12]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x22]\n" ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" - ".inst 0xe0160700 // ld1b { za0h.b[x12] }, p1/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" + "ldr x25, [x26, #0x0]\n" ".inst 0xe01602e4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x23, x22]\n" "add x12, x12, #0x8\n" + "cmp x12, x28, LSL #2\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "cmp x12, x28, LSL #2\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25246141 // psel p1.b, p8.b/Z, p10.b[w12]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x22]\n" ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" "mov x26, %x[in]\n" - "incb x27\n" - ".inst 0xe0160700 // ld1b { za0h.b[x12] }, p1/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" ".inst 0xe01602e4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x23, x22]\n" + "ldr x25, [x26, #0x0]\n" + "incb x22\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "incb x22\n" + "incb x27\n" "cbz x20, 8f\n" "mov x20, x20\n" "3:" // K loop: Main loop "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - "mov x14, #0x0\n" "cbz x28, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25346143 // psel p3.b, p8.b/Z, p10.b[w12, #2]\n" - ".inst 0x25746142 // psel p2.b, p8.b/Z, p10.b[w12, #6]\n" - ".inst 0x25266d21 // psel p1.b, p11.b/Z, p9.b[w14]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - ".inst 0xe0160f02 // ld1b { za0h.b[x12, #2] }, p3/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0160ae6 // ld1b { za0h.b[x12, #6] }, p2/Z, [x23, x22]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x23, x22]\n" "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0a982a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" "add x26, x26, #0x10\n" - "add x12, x12, #0x8\n" - ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" - ".inst 0xe0a9c2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x9, LSL #2]\n" - "add x14, x14, #0x2\n" "addvl x21, x21, #2\n" - "cmp x14, x28\n" + "add x13, x13, #0x8\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25346143 // psel p3.b, p8.b/Z, p10.b[w12, #2]\n" - ".inst 0x25746142 // psel p2.b, p8.b/Z, p10.b[w12, #6]\n" - ".inst 0x25266d21 // psel p1.b, p11.b/Z, p9.b[w14]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "whilelt p9.b, x27, %x[width]\n" - ".inst 0xe0160f02 // ld1b { za0h.b[x12, #2] }, p3/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" "incb x27\n" - "mov x13, #0x0\n" - ".inst 0xe0160ae6 // ld1b { za0h.b[x12, #6] }, p2/Z, [x23, x22]\n" - "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" + ".inst 0xe0a982a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "addvl x21, x21, #2\n" "incb x22\n" - ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - ".inst 0xe0a9c2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x9, LSL #2]\n" - "addvl x21, x21, #2\n" "cbz x28, 7f\n" "6:" // K loop: Main loop: Second: Loop - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x25656142 // psel p2.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x25246d21 // psel p1.b, p11.b/Z, p9.b[w12]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - ".inst 0xe0162f00 // ld1b { za0h.b[x13] }, p3/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0162ae4 // ld1b { za0h.b[x13, #4] }, p2/Z, [x23, x22]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162320 // ld1b { za0h.b[x13] }, p0/Z, [x25, x22]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x23, x22]\n" "ldr x23, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - "add x13, x13, #0x8\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" ".inst 0xe0a982a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" "add x12, x12, #0x2\n" - "addvl x21, x21, #2\n" "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x8\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x25656142 // psel p2.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x25246d21 // psel p1.b, p11.b/Z, p9.b[w12]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162320 // ld1b { za0h.b[x13] }, p0/Z, [x25, x22]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "whilelt p9.b, x27, %x[width]\n" - ".inst 0xe0162f00 // ld1b { za0h.b[x13] }, p3/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" "subs x20, x20, #0x1\n" - "incb x27\n" - ".inst 0xe0162ae4 // ld1b { za0h.b[x13, #4] }, p2/Z, [x23, x22]\n" - "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "incb x22\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" ".inst 0xe0a982a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" "addvl x21, x21, #2\n" + "incb x27\n" + "incb x22\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x25, 11f\n" + "cbnz x24, 11f\n" "mov x26, %x[in]\n" "whilelt p8.b, x27, %x[width]\n" "mov x13, #0x0\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" - "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "ldr x20, [x26, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" "cmp x12, x9\n" - "add x26, x26, #0x8\n" ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" "add x13, x13, #0x4\n" "blt 9b\n" "whilelt p9.b, x27, %x[width]\n" @@ -192,11 +192,11 @@ void interleave_block<1, 4, VLType::SME, false>( "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - "add x20, x20, #0x4\n" ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "add x20, x20, #0x4\n" "blt 10b\n" "whilelt p8.b, x27, %x[width]\n" "b 13f\n" @@ -206,15 +206,15 @@ void interleave_block<1, 4, VLType::SME, false>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x10\n" + "addvl x21, x21, #1\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x21\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp index ba8be81ade..fe98bd86b5 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp @@ -32,18 +32,18 @@ void interleave_block<1, 4, VLType::SME, true>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" "mov x23, %x[width]\n" - "mov x21, %x[width]\n" - "cntb x20\n" - "incb x23\n" "mov z18.b, #0x1\n" + "incb x23\n" + "mov x20, %x[width]\n" "mov z17.s, #0x0\n" - "sub x10, x20, #0x1\n" + "sub x10, x21, #0x1\n" "cntw x9\n" "sub x23, x23, #0x1\n" - "ands x10, x21, x10\n" - "udiv x23, x23, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x10, x10, x20, NE\n" + "ands x10, x20, x10\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x10, x10, x21, NE\n" "lsl x22, %x[height], #0x1\n" // height * 2 "lsl x21, x9, #0x1\n" "sub x20, x23, #0x1\n" @@ -51,7 +51,7 @@ void interleave_block<1, 4, VLType::SME, true>( "whilelt p9.b, XZR, x22\n" "whilelt p8.b, x21, x22\n" "mov x28, #0x0\n" - "ptrue p4.b\n" + "ptrue p2.b\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 "and x27, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) "lsr x10, x10, #0x2\n" @@ -64,132 +64,132 @@ void interleave_block<1, 4, VLType::SME, true>( "whilelt p8.b, x28, %x[width]\n" "cbnz %x[first], 1f\n" "addvl x24, x24, #-1\n" - "ld1w { z17.s }, p4/Z, [x24]\n" + "ld1w { z17.s }, p2/Z, [x24]\n" "1:" // K loop: Load row sums: End "mov x23, %x[in]\n" - "mov x12, #0x0\n" "ldr x22, [x23, #0x0]\n" + "mov x12, #0x0\n" "ldr x21, [x23, #0x8]\n" "add x23, x23, #0x10\n" "cbz x26, 3f\n" "2:" // K loop: Charge: Loop - ".inst 0x25246141 // psel p1.b, p8.b/Z, p10.b[w12]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902c0 // ld1b { za0h.b[x12] }, p0/Z, [x22, x25]\n" ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" - ".inst 0xe01906c0 // ld1b { za0h.b[x12] }, p1/Z, [x22, x25]\n" "ldr x22, [x23, #0x0]\n" ".inst 0xe01902a4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x21, x25]\n" "add x12, x12, #0x8\n" + "cmp x12, x26, LSL #2\n" "ldr x21, [x23, #0x8]\n" "add x23, x23, #0x10\n" - "cmp x12, x26, LSL #2\n" "blt 2b\n" "3:" // K loop: Charge: End - ".inst 0x25246141 // psel p1.b, p8.b/Z, p10.b[w12]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902c0 // ld1b { za0h.b[x12] }, p0/Z, [x22, x25]\n" ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" "mov x23, %x[in]\n" - "incb x28\n" - ".inst 0xe01906c0 // ld1b { za0h.b[x12] }, p1/Z, [x22, x25]\n" - "ldr x22, [x23, #0x0]\n" ".inst 0xe01902a4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x21, x25]\n" + "ldr x22, [x23, #0x0]\n" + "incb x25\n" "ldr x21, [x23, #0x8]\n" "add x23, x23, #0x10\n" - "incb x25\n" + "incb x28\n" "cbz x20, 9f\n" "mov x20, x20\n" "4:" // K loop: Main loop "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - "mov x14, #0x0\n" "cbz x26, 6f\n" "5:" // K loop: Main loop: First: Loop - ".inst 0x25346143 // psel p3.b, p8.b/Z, p10.b[w12, #2]\n" - ".inst 0x25746142 // psel p2.b, p8.b/Z, p10.b[w12, #6]\n" - ".inst 0x25266d21 // psel p1.b, p11.b/Z, p9.b[w14]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - ".inst 0xe0190ec2 // ld1b { za0h.b[x12, #2] }, p3/Z, [x22, x25]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" "ldr x22, [x23, #0x0]\n" - ".inst 0xe0190aa6 // ld1b { za0h.b[x12, #6] }, p2/Z, [x21, x25]\n" + ".inst 0xe01922a6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + "sdot z17.s, z16.b, z18.b\n" "ldr x21, [x23, #0x8]\n" - "add x23, x23, #0x10\n" - "add x12, x12, #0x8\n" - ".inst 0xc082d010 // mova z16.s, p4/M, za0v.s[x14]\n" - ".inst 0xe0bfc700 // st1w { za0v.s[x14] }, p1/Z, [x24, XZR, LSL #2]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828830 // mova z16.s, p2/M, za0v.s[x12, #1]\n" + ".inst 0xe0a98301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" "sdot z17.s, z16.b, z18.b\n" - ".inst 0xe0a9c301 // st1w { za0v.s[x14, #1] }, p0/Z, [x24, x9, LSL #2]\n" - ".inst 0xc082d030 // mova z16.s, p4/M, za0v.s[x14, #1]\n" - "add x14, x14, #0x2\n" - "cmp x14, x26\n" + "add x23, x23, #0x10\n" "addvl x24, x24, #2\n" - "sdot z17.s, z16.b, z18.b\n" + "add x13, x13, #0x8\n" "blt 5b\n" "6:" // K loop: Main loop: First: Tail - ".inst 0x25346143 // psel p3.b, p8.b/Z, p10.b[w12, #2]\n" - ".inst 0x25746142 // psel p2.b, p8.b/Z, p10.b[w12, #6]\n" - ".inst 0x25266d21 // psel p1.b, p11.b/Z, p9.b[w14]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01922a6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "sdot z17.s, z16.b, z18.b\n" "mov x23, %x[in]\n" - "whilelt p9.b, x28, %x[width]\n" - ".inst 0xe0190ec2 // ld1b { za0h.b[x12, #2] }, p3/Z, [x22, x25]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x23, #0x0]\n" - "incb x28\n" - "mov x13, #0x0\n" - ".inst 0xe0190aa6 // ld1b { za0h.b[x12, #6] }, p2/Z, [x21, x25]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828830 // mova z16.s, p2/M, za0v.s[x12, #1]\n" "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8700 // st1w { za0v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" + "whilelt p9.b, x28, %x[width]\n" + "incb x28\n" "add x23, x23, #0x10\n" + ".inst 0xe0a98301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "sdot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #2\n" "incb x25\n" - ".inst 0xc082d010 // mova z16.s, p4/M, za0v.s[x14]\n" - ".inst 0xe0bfc700 // st1w { za0v.s[x14] }, p1/Z, [x24, XZR, LSL #2]\n" "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - "sdot z17.s, z16.b, z18.b\n" - ".inst 0xc082d030 // mova z16.s, p4/M, za0v.s[x14, #1]\n" - ".inst 0xe0a9c301 // st1w { za0v.s[x14, #1] }, p0/Z, [x24, x9, LSL #2]\n" - "addvl x24, x24, #2\n" - "sdot z17.s, z16.b, z18.b\n" "cbz x26, 8f\n" "7:" // K loop: Main loop: Second: Loop - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x25656142 // psel p2.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x25246d21 // psel p1.b, p11.b/Z, p9.b[w12]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - ".inst 0xe0192ec0 // ld1b { za0h.b[x13] }, p3/Z, [x22, x25]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x25]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" "ldr x22, [x23, #0x0]\n" - ".inst 0xe0192aa4 // ld1b { za0h.b[x13, #4] }, p2/Z, [x21, x25]\n" - "ldr x21, [x23, #0x8]\n" - "add x23, x23, #0x10\n" - "add x13, x13, #0x8\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" - ".inst 0xe0bf8708 // st1w { za2v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" + ".inst 0xe01922a4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" "sdot z17.s, z16.b, z18.b\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828930 // mova z16.s, p2/M, za2v.s[x12, #1]\n" ".inst 0xe0a98309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" - ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" "add x12, x12, #0x2\n" "cmp x12, x26\n" - "addvl x24, x24, #2\n" "sdot z17.s, z16.b, z18.b\n" + "add x23, x23, #0x10\n" + "addvl x24, x24, #2\n" + "add x13, x13, #0x8\n" "blt 7b\n" "8:" // K loop: Main loop: Second: Tail - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x25656142 // psel p2.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x25246d21 // psel p1.b, p11.b/Z, p9.b[w12]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x25]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01922a4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + "sdot z17.s, z16.b, z18.b\n" "mov x23, %x[in]\n" - "whilelt p9.b, x28, %x[width]\n" - ".inst 0xe0192ec0 // ld1b { za0h.b[x13] }, p3/Z, [x22, x25]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x23, #0x0]\n" - "subs x20, x20, #0x1\n" - "incb x28\n" - ".inst 0xe0192aa4 // ld1b { za0h.b[x13, #4] }, p2/Z, [x21, x25]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828930 // mova z16.s, p2/M, za2v.s[x12, #1]\n" "ldr x21, [x23, #0x8]\n" - "add x23, x23, #0x10\n" - "incb x25\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" ".inst 0xe0bf8708 // st1w { za2v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" - "sdot z17.s, z16.b, z18.b\n" - ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" + "whilelt p9.b, x28, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x23, x23, #0x10\n" ".inst 0xe0a98309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" - "addvl x24, x24, #2\n" "sdot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #2\n" + "incb x28\n" + "incb x25\n" "bgt 4b\n" "9:" // K loop: Tails "cbnz x27, 12f\n" @@ -198,17 +198,17 @@ void interleave_block<1, 4, VLType::SME, true>( "mov x13, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: First - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "ldr x20, [x23, #0x0]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" "sdot z17.s, z16.b, z18.b\n" - ".inst 0xe0bf8700 // st1w { za0v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" - "add x12, x12, #0x1\n" - "addvl x24, x24, #1\n" - "ldr x20, [x23, #0x0]\n" + ".inst 0xe0192282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x25]\n" "cmp x12, x9\n" "add x23, x23, #0x8\n" - ".inst 0xe0192282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x25]\n" + "addvl x24, x24, #1\n" "add x13, x13, #0x4\n" "blt 10b\n" "whilelt p9.b, x28, %x[width]\n" @@ -217,13 +217,13 @@ void interleave_block<1, 4, VLType::SME, true>( "mov x12, #0x0\n" "11:" // K loop: Tails: Even: Second ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" - "add x20, x20, #0x4\n" - "sdot z17.s, z16.b, z18.b\n" ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" "add x12, x12, #0x1\n" - "addvl x24, x24, #1\n" "cmp x12, x10\n" + "sdot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #1\n" + "add x20, x20, #0x4\n" "blt 11b\n" "whilelt p8.b, x28, %x[width]\n" "b 14f\n" @@ -231,21 +231,21 @@ void interleave_block<1, 4, VLType::SME, true>( "mov x12, #0x0\n" "13:" // K loop: Tails: Odd: Loop ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" - "sdot z17.s, z16.b, z18.b\n" ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" "add x12, x12, #0x1\n" - "addvl x24, x24, #1\n" "cmp x12, x10\n" + "sdot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #1\n" "blt 13b\n" "14:" // K loop: End - "st1w { z17.s }, p4, [x24]\n" + "st1w { z17.s }, p2, [x24]\n" "addvl x24, x24, #1\n" - ".inst 0xd503467f // SMSTOP\n" "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp index 6d1c1a207f..bc7d013798 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp @@ -32,16 +32,16 @@ void interleave_block<1, 4, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" "mov x23, %x[width]\n" - "mov x21, %x[width]\n" - "cntb x20\n" "incb x23\n" - "sub x10, x20, #0x1\n" + "mov x20, %x[width]\n" + "sub x10, x21, #0x1\n" "cntw x9\n" "sub x23, x23, #0x1\n" - "ands x10, x21, x10\n" - "udiv x23, x23, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x10, x10, x20, NE\n" + "ands x10, x20, x10\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x10, x10, x21, NE\n" "lsl x22, %x[height], #0x1\n" // height * 2 "lsl x21, x9, #0x1\n" "sub x20, x23, #0x1\n" @@ -52,12 +52,12 @@ void interleave_block<1, 4, VLType::SME, false>( "mov x27, #0x0\n" "mov x26, %x[in]\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x25, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x25, [x26, #0x0]\n" + "and x24, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) "lsr x10, x10, #0x2\n" - "ldr x24, [x26, #0x0]\n" + "ldr x23, [x26, #0x8]\n" "ptrue p11.s\n" "zip1 p10.b, p9.b, p8.b\n" - "ldr x23, [x26, #0x8]\n" "mov x22, %x[row_offset]\n" "mov x21, %x[out]\n" "whilelt p9.b, x27, %x[width]\n" @@ -66,124 +66,124 @@ void interleave_block<1, 4, VLType::SME, false>( "mov x12, #0x0\n" "cbz x28, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25246141 // psel p1.b, p8.b/Z, p10.b[w12]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x22]\n" ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" - ".inst 0xe0160700 // ld1b { za0h.b[x12] }, p1/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" + "ldr x25, [x26, #0x0]\n" ".inst 0xe01602e4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x23, x22]\n" "add x12, x12, #0x8\n" + "cmp x12, x28, LSL #2\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "cmp x12, x28, LSL #2\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25246141 // psel p1.b, p8.b/Z, p10.b[w12]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x22]\n" ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" "mov x26, %x[in]\n" - "incb x27\n" - ".inst 0xe0160700 // ld1b { za0h.b[x12] }, p1/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" ".inst 0xe01602e4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x23, x22]\n" + "ldr x25, [x26, #0x0]\n" + "incb x22\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "incb x22\n" + "incb x27\n" "cbz x20, 8f\n" "mov x20, x20\n" "3:" // K loop: Main loop "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - "mov x14, #0x0\n" "cbz x28, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25346143 // psel p3.b, p8.b/Z, p10.b[w12, #2]\n" - ".inst 0x25746142 // psel p2.b, p8.b/Z, p10.b[w12, #6]\n" - ".inst 0x25266d21 // psel p1.b, p11.b/Z, p9.b[w14]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - ".inst 0xe0160f02 // ld1b { za0h.b[x12, #2] }, p3/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0160ae6 // ld1b { za0h.b[x12, #6] }, p2/Z, [x23, x22]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x23, x22]\n" "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0a982a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" "add x26, x26, #0x10\n" - "add x12, x12, #0x8\n" - ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" - ".inst 0xe0a9c2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x9, LSL #2]\n" - "add x14, x14, #0x2\n" "addvl x21, x21, #2\n" - "cmp x14, x28\n" + "add x13, x13, #0x8\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25346143 // psel p3.b, p8.b/Z, p10.b[w12, #2]\n" - ".inst 0x25746142 // psel p2.b, p8.b/Z, p10.b[w12, #6]\n" - ".inst 0x25266d21 // psel p1.b, p11.b/Z, p9.b[w14]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "whilelt p9.b, x27, %x[width]\n" - ".inst 0xe0160f02 // ld1b { za0h.b[x12, #2] }, p3/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" "incb x27\n" - "mov x13, #0x0\n" - ".inst 0xe0160ae6 // ld1b { za0h.b[x12, #6] }, p2/Z, [x23, x22]\n" - "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" + ".inst 0xe0a982a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "addvl x21, x21, #2\n" "incb x22\n" - ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - ".inst 0xe0a9c2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x9, LSL #2]\n" - "addvl x21, x21, #2\n" "cbz x28, 7f\n" "6:" // K loop: Main loop: Second: Loop - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x25656142 // psel p2.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x25246d21 // psel p1.b, p11.b/Z, p9.b[w12]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - ".inst 0xe0162f00 // ld1b { za0h.b[x13] }, p3/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0162ae4 // ld1b { za0h.b[x13, #4] }, p2/Z, [x23, x22]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162320 // ld1b { za0h.b[x13] }, p0/Z, [x25, x22]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x23, x22]\n" "ldr x23, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - "add x13, x13, #0x8\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" ".inst 0xe0a982a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" "add x12, x12, #0x2\n" - "addvl x21, x21, #2\n" "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x8\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x25656142 // psel p2.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x25246d21 // psel p1.b, p11.b/Z, p9.b[w12]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162320 // ld1b { za0h.b[x13] }, p0/Z, [x25, x22]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "whilelt p9.b, x27, %x[width]\n" - ".inst 0xe0162f00 // ld1b { za0h.b[x13] }, p3/Z, [x24, x22]\n" - "ldr x24, [x26, #0x0]\n" "subs x20, x20, #0x1\n" - "incb x27\n" - ".inst 0xe0162ae4 // ld1b { za0h.b[x13, #4] }, p2/Z, [x23, x22]\n" - "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "incb x22\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" ".inst 0xe0a982a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" "addvl x21, x21, #2\n" + "incb x27\n" + "incb x22\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x25, 11f\n" + "cbnz x24, 11f\n" "mov x26, %x[in]\n" "whilelt p8.b, x27, %x[width]\n" "mov x13, #0x0\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" - "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "ldr x20, [x26, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" "cmp x12, x9\n" - "add x26, x26, #0x8\n" ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" "add x13, x13, #0x4\n" "blt 9b\n" "whilelt p9.b, x27, %x[width]\n" @@ -192,11 +192,11 @@ void interleave_block<1, 4, VLType::SME, false>( "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - "add x20, x20, #0x4\n" ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "add x20, x20, #0x4\n" "blt 10b\n" "whilelt p8.b, x27, %x[width]\n" "b 13f\n" @@ -206,15 +206,15 @@ void interleave_block<1, 4, VLType::SME, false>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x10\n" + "addvl x21, x21, #1\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x21\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp index dbcd18678b..66fcd800d4 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp @@ -32,18 +32,18 @@ void interleave_block<1, 4, VLType::SME, true>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" "mov x23, %x[width]\n" - "mov x21, %x[width]\n" - "cntb x20\n" - "incb x23\n" "mov z18.b, #0x1\n" + "incb x23\n" + "mov x20, %x[width]\n" "mov z17.s, #0x0\n" - "sub x10, x20, #0x1\n" + "sub x10, x21, #0x1\n" "cntw x9\n" "sub x23, x23, #0x1\n" - "ands x10, x21, x10\n" - "udiv x23, x23, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x10, x10, x20, NE\n" + "ands x10, x20, x10\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x10, x10, x21, NE\n" "lsl x22, %x[height], #0x1\n" // height * 2 "lsl x21, x9, #0x1\n" "sub x20, x23, #0x1\n" @@ -51,7 +51,7 @@ void interleave_block<1, 4, VLType::SME, true>( "whilelt p9.b, XZR, x22\n" "whilelt p8.b, x21, x22\n" "mov x28, #0x0\n" - "ptrue p4.b\n" + "ptrue p2.b\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 "and x27, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) "lsr x10, x10, #0x2\n" @@ -64,132 +64,132 @@ void interleave_block<1, 4, VLType::SME, true>( "whilelt p8.b, x28, %x[width]\n" "cbnz %x[first], 1f\n" "addvl x24, x24, #-1\n" - "ld1w { z17.s }, p4/Z, [x24]\n" + "ld1w { z17.s }, p2/Z, [x24]\n" "1:" // K loop: Load row sums: End "mov x23, %x[in]\n" - "mov x12, #0x0\n" "ldr x22, [x23, #0x0]\n" + "mov x12, #0x0\n" "ldr x21, [x23, #0x8]\n" "add x23, x23, #0x10\n" "cbz x26, 3f\n" "2:" // K loop: Charge: Loop - ".inst 0x25246141 // psel p1.b, p8.b/Z, p10.b[w12]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902c0 // ld1b { za0h.b[x12] }, p0/Z, [x22, x25]\n" ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" - ".inst 0xe01906c0 // ld1b { za0h.b[x12] }, p1/Z, [x22, x25]\n" "ldr x22, [x23, #0x0]\n" ".inst 0xe01902a4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x21, x25]\n" "add x12, x12, #0x8\n" + "cmp x12, x26, LSL #2\n" "ldr x21, [x23, #0x8]\n" "add x23, x23, #0x10\n" - "cmp x12, x26, LSL #2\n" "blt 2b\n" "3:" // K loop: Charge: End - ".inst 0x25246141 // psel p1.b, p8.b/Z, p10.b[w12]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902c0 // ld1b { za0h.b[x12] }, p0/Z, [x22, x25]\n" ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" "mov x23, %x[in]\n" - "incb x28\n" - ".inst 0xe01906c0 // ld1b { za0h.b[x12] }, p1/Z, [x22, x25]\n" - "ldr x22, [x23, #0x0]\n" ".inst 0xe01902a4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x21, x25]\n" + "ldr x22, [x23, #0x0]\n" + "incb x25\n" "ldr x21, [x23, #0x8]\n" "add x23, x23, #0x10\n" - "incb x25\n" + "incb x28\n" "cbz x20, 9f\n" "mov x20, x20\n" "4:" // K loop: Main loop "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - "mov x14, #0x0\n" "cbz x26, 6f\n" "5:" // K loop: Main loop: First: Loop - ".inst 0x25346143 // psel p3.b, p8.b/Z, p10.b[w12, #2]\n" - ".inst 0x25746142 // psel p2.b, p8.b/Z, p10.b[w12, #6]\n" - ".inst 0x25266d21 // psel p1.b, p11.b/Z, p9.b[w14]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - ".inst 0xe0190ec2 // ld1b { za0h.b[x12, #2] }, p3/Z, [x22, x25]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" "ldr x22, [x23, #0x0]\n" - ".inst 0xe0190aa6 // ld1b { za0h.b[x12, #6] }, p2/Z, [x21, x25]\n" + ".inst 0xe01922a6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + "udot z17.s, z16.b, z18.b\n" "ldr x21, [x23, #0x8]\n" - "add x23, x23, #0x10\n" - "add x12, x12, #0x8\n" - ".inst 0xc082d010 // mova z16.s, p4/M, za0v.s[x14]\n" - ".inst 0xe0bfc700 // st1w { za0v.s[x14] }, p1/Z, [x24, XZR, LSL #2]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828830 // mova z16.s, p2/M, za0v.s[x12, #1]\n" + ".inst 0xe0a98301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" "udot z17.s, z16.b, z18.b\n" - ".inst 0xe0a9c301 // st1w { za0v.s[x14, #1] }, p0/Z, [x24, x9, LSL #2]\n" - ".inst 0xc082d030 // mova z16.s, p4/M, za0v.s[x14, #1]\n" - "add x14, x14, #0x2\n" - "cmp x14, x26\n" + "add x23, x23, #0x10\n" "addvl x24, x24, #2\n" - "udot z17.s, z16.b, z18.b\n" + "add x13, x13, #0x8\n" "blt 5b\n" "6:" // K loop: Main loop: First: Tail - ".inst 0x25346143 // psel p3.b, p8.b/Z, p10.b[w12, #2]\n" - ".inst 0x25746142 // psel p2.b, p8.b/Z, p10.b[w12, #6]\n" - ".inst 0x25266d21 // psel p1.b, p11.b/Z, p9.b[w14]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01922a6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "udot z17.s, z16.b, z18.b\n" "mov x23, %x[in]\n" - "whilelt p9.b, x28, %x[width]\n" - ".inst 0xe0190ec2 // ld1b { za0h.b[x12, #2] }, p3/Z, [x22, x25]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x23, #0x0]\n" - "incb x28\n" - "mov x13, #0x0\n" - ".inst 0xe0190aa6 // ld1b { za0h.b[x12, #6] }, p2/Z, [x21, x25]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828830 // mova z16.s, p2/M, za0v.s[x12, #1]\n" "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8700 // st1w { za0v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" + "whilelt p9.b, x28, %x[width]\n" + "incb x28\n" "add x23, x23, #0x10\n" + ".inst 0xe0a98301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "udot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #2\n" "incb x25\n" - ".inst 0xc082d010 // mova z16.s, p4/M, za0v.s[x14]\n" - ".inst 0xe0bfc700 // st1w { za0v.s[x14] }, p1/Z, [x24, XZR, LSL #2]\n" "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" "mov x12, #0x0\n" - "udot z17.s, z16.b, z18.b\n" - ".inst 0xc082d030 // mova z16.s, p4/M, za0v.s[x14, #1]\n" - ".inst 0xe0a9c301 // st1w { za0v.s[x14, #1] }, p0/Z, [x24, x9, LSL #2]\n" - "addvl x24, x24, #2\n" - "udot z17.s, z16.b, z18.b\n" "cbz x26, 8f\n" "7:" // K loop: Main loop: Second: Loop - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x25656142 // psel p2.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x25246d21 // psel p1.b, p11.b/Z, p9.b[w12]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - ".inst 0xe0192ec0 // ld1b { za0h.b[x13] }, p3/Z, [x22, x25]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x25]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" "ldr x22, [x23, #0x0]\n" - ".inst 0xe0192aa4 // ld1b { za0h.b[x13, #4] }, p2/Z, [x21, x25]\n" - "ldr x21, [x23, #0x8]\n" - "add x23, x23, #0x10\n" - "add x13, x13, #0x8\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" - ".inst 0xe0bf8708 // st1w { za2v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" + ".inst 0xe01922a4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" "udot z17.s, z16.b, z18.b\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828930 // mova z16.s, p2/M, za2v.s[x12, #1]\n" ".inst 0xe0a98309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" - ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" "add x12, x12, #0x2\n" "cmp x12, x26\n" - "addvl x24, x24, #2\n" "udot z17.s, z16.b, z18.b\n" + "add x23, x23, #0x10\n" + "addvl x24, x24, #2\n" + "add x13, x13, #0x8\n" "blt 7b\n" "8:" // K loop: Main loop: Second: Tail - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x25656142 // psel p2.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x25246d21 // psel p1.b, p11.b/Z, p9.b[w12]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x25]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01922a4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + "udot z17.s, z16.b, z18.b\n" "mov x23, %x[in]\n" - "whilelt p9.b, x28, %x[width]\n" - ".inst 0xe0192ec0 // ld1b { za0h.b[x13] }, p3/Z, [x22, x25]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x23, #0x0]\n" - "subs x20, x20, #0x1\n" - "incb x28\n" - ".inst 0xe0192aa4 // ld1b { za0h.b[x13, #4] }, p2/Z, [x21, x25]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828930 // mova z16.s, p2/M, za2v.s[x12, #1]\n" "ldr x21, [x23, #0x8]\n" - "add x23, x23, #0x10\n" - "incb x25\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" ".inst 0xe0bf8708 // st1w { za2v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" - "udot z17.s, z16.b, z18.b\n" - ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" + "whilelt p9.b, x28, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x23, x23, #0x10\n" ".inst 0xe0a98309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" - "addvl x24, x24, #2\n" "udot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #2\n" + "incb x28\n" + "incb x25\n" "bgt 4b\n" "9:" // K loop: Tails "cbnz x27, 12f\n" @@ -198,17 +198,17 @@ void interleave_block<1, 4, VLType::SME, true>( "mov x13, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: First - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "ldr x20, [x23, #0x0]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" "udot z17.s, z16.b, z18.b\n" - ".inst 0xe0bf8700 // st1w { za0v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" - "add x12, x12, #0x1\n" - "addvl x24, x24, #1\n" - "ldr x20, [x23, #0x0]\n" + ".inst 0xe0192282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x25]\n" "cmp x12, x9\n" "add x23, x23, #0x8\n" - ".inst 0xe0192282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x25]\n" + "addvl x24, x24, #1\n" "add x13, x13, #0x4\n" "blt 10b\n" "whilelt p9.b, x28, %x[width]\n" @@ -217,13 +217,13 @@ void interleave_block<1, 4, VLType::SME, true>( "mov x12, #0x0\n" "11:" // K loop: Tails: Even: Second ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" - "add x20, x20, #0x4\n" - "udot z17.s, z16.b, z18.b\n" ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" "add x12, x12, #0x1\n" - "addvl x24, x24, #1\n" "cmp x12, x10\n" + "udot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #1\n" + "add x20, x20, #0x4\n" "blt 11b\n" "whilelt p8.b, x28, %x[width]\n" "b 14f\n" @@ -231,21 +231,21 @@ void interleave_block<1, 4, VLType::SME, true>( "mov x12, #0x0\n" "13:" // K loop: Tails: Odd: Loop ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" - "udot z17.s, z16.b, z18.b\n" ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" "add x12, x12, #0x1\n" - "addvl x24, x24, #1\n" "cmp x12, x10\n" + "udot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #1\n" "blt 13b\n" "14:" // K loop: End - "st1w { z17.s }, p4, [x24]\n" + "st1w { z17.s }, p2, [x24]\n" "addvl x24, x24, #1\n" - ".inst 0xd503467f // SMSTOP\n" "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp index 591c08dcb2..fb0a74b3bd 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp @@ -33,25 +33,25 @@ void interleave_block<1, 1, VLType::SME, false>( __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "mov x21, %x[width]\n" - "mov x20, %x[width]\n" "inch x21\n" "cnth x11\n" "sub x21, x21, #0x1\n" - "sub x10, x11, #0x1\n" "udiv x21, x21, x11\n" // n_passes = ceildiv(width, VL<T>) - "ands x10, x20, x10\n" + "mov x20, %x[width]\n" + "sub x10, x11, #0x1\n" "sub x9, x21, #0x1\n" + "ands x10, x20, x10\n" "sub x28, x11, #0x2\n" "lsl x20, %x[height], #0x1\n" // height * 2 "mov x27, #0x0\n" "mov x26, %x[in]\n" "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x25, [x26, #0x0]\n" + "and x24, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) "csel x10, x10, x11, NE\n" - "ldr x24, [x26, #0x0]\n" + "ldr x23, [x26, #0x8]\n" "ptrue p11.h\n" "whilelt p10.h, XZR, x20\n" - "ldr x23, [x26, #0x8]\n" "mov x22, %x[row_offset]\n" "mov x21, %x[out]\n" "whilelt p9.h, x27, %x[width]\n" @@ -60,119 +60,119 @@ void interleave_block<1, 1, VLType::SME, false>( "mov x12, #0x0\n" "cbz x28, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25286141 // psel p1.h, p8.h/Z, p10.h[w12]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0xe0560700 // ld1h { za0h.h[x12] }, p1/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" + "ldr x25, [x26, #0x0]\n" ".inst 0xe05602e1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x23, x22, LSL #1]\n" "add x12, x12, #0x2\n" + "cmp x12, x28\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "cmp x12, x28\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25286141 // psel p1.h, p8.h/Z, p10.h[w12]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" "mov x26, %x[in]\n" - "inch x27\n" - ".inst 0xe0560700 // ld1h { za0h.h[x12] }, p1/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" ".inst 0xe05602e1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x23, x22, LSL #1]\n" + "ldr x25, [x26, #0x0]\n" + "inch x22\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "inch x22\n" + "inch x27\n" "cbz x9, 8f\n" "mov x20, x9\n" "3:" // K loop: Main loop "whilelt p8.h, x27, %x[width]\n" - "mov x13, #0x0\n" + "mov x12, #0x0\n" "cbz x28, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25296143 // psel p3.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25396142 // psel p2.h, p8.h/Z, p10.h[w13, #1]\n" - ".inst 0x25296d21 // psel p1.h, p11.h/Z, p9.h[w13]\n" - ".inst 0x25396d20 // psel p0.h, p11.h/Z, p9.h[w13, #1]\n" - ".inst 0xe0562f08 // ld1h { za1h.h[x13] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0562ae9 // ld1h { za1h.h[x13, #1] }, p2/Z, [x23, x22, LSL #1]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" "ldr x23, [x26, #0x8]\n" - ".inst 0xe07fa6a0 // st1h { za0v.h[x13] }, p1/Z, [x21, XZR, LSL #1]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06b82a1 // st1h { za0v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" "add x26, x26, #0x10\n" - ".inst 0xe06ba2a1 // st1h { za0v.h[x13, #1] }, p0/Z, [x21, x11, LSL #1]\n" - "add x13, x13, #0x2\n" "addvl x21, x21, #2\n" - "cmp x13, x28\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25296143 // psel p3.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25396142 // psel p2.h, p8.h/Z, p10.h[w13, #1]\n" - ".inst 0x25296d21 // psel p1.h, p11.h/Z, p9.h[w13]\n" - ".inst 0x25396d20 // psel p0.h, p11.h/Z, p9.h[w13, #1]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" "whilelt p9.h, x27, %x[width]\n" - ".inst 0xe0562f08 // ld1h { za1h.h[x13] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" "inch x27\n" - "mov x12, #0x0\n" - ".inst 0xe0562ae9 // ld1h { za1h.h[x13, #1] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x26, #0x8]\n" - ".inst 0xe07fa6a0 // st1h { za0v.h[x13] }, p1/Z, [x21, XZR, LSL #1]\n" "add x26, x26, #0x10\n" - ".inst 0xe06ba2a1 // st1h { za0v.h[x13, #1] }, p0/Z, [x21, x11, LSL #1]\n" + ".inst 0xe06b82a1 // st1h { za0v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" "addvl x21, x21, #2\n" "inch x22\n" "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" "cbz x28, 7f\n" "6:" // K loop: Main loop: Second: Loop - ".inst 0x25286143 // psel p3.h, p8.h/Z, p10.h[w12]\n" - ".inst 0x25386142 // psel p2.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" - ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" - ".inst 0xe0560f00 // ld1h { za0h.h[x12] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0560ae1 // ld1h { za0h.h[x12, #1] }, p2/Z, [x23, x22, LSL #1]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" "ldr x23, [x26, #0x8]\n" - ".inst 0xe07f86a8 // st1h { za1v.h[x12] }, p1/Z, [x21, XZR, LSL #1]\n" - "add x26, x26, #0x10\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" ".inst 0xe06b82a9 // st1h { za1v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" "add x12, x12, #0x2\n" - "addvl x21, x21, #2\n" "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail - ".inst 0x25286143 // psel p3.h, p8.h/Z, p10.h[w12]\n" - ".inst 0x25386142 // psel p2.h, p8.h/Z, p10.h[w12, #1]\n" - ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" - ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" "whilelt p9.h, x27, %x[width]\n" - ".inst 0xe0560f00 // ld1h { za0h.h[x12] }, p3/Z, [x24, x22, LSL #1]\n" - "ldr x24, [x26, #0x0]\n" "subs x20, x20, #0x1\n" - "inch x27\n" - ".inst 0xe0560ae1 // ld1h { za0h.h[x12, #1] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x26, #0x8]\n" - ".inst 0xe07f86a8 // st1h { za1v.h[x12] }, p1/Z, [x21, XZR, LSL #1]\n" "add x26, x26, #0x10\n" ".inst 0xe06b82a9 // st1h { za1v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" "addvl x21, x21, #2\n" + "inch x27\n" "inch x22\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x25, 11f\n" + "cbnz x24, 11f\n" "mov x26, %x[in]\n" "whilelt p8.h, x27, %x[width]\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" - ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" - ".inst 0xe07f86a0 // st1h { za0v.h[x12] }, p1/Z, [x21, XZR, LSL #1]\n" - "addvl x21, x21, #1\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" "ldr x20, [x26, #0x0]\n" - "add x26, x26, #0x8\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" ".inst 0xe0560288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x22, LSL #1]\n" "add x12, x12, #0x1\n" "cmp x12, x11\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" "blt 9b\n" "whilelt p9.h, x27, %x[width]\n" "whilelt p8.h, x27, %x[width]\n" @@ -181,8 +181,8 @@ void interleave_block<1, 1, VLType::SME, false>( ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x10\n" + "addvl x21, x21, #1\n" "blt 10b\n" "whilelt p8.h, x27, %x[width]\n" "b 13f\n" @@ -192,15 +192,15 @@ void interleave_block<1, 1, VLType::SME, false>( ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x10\n" + "addvl x21, x21, #1\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x21\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp index b76ec57d22..3fe3885068 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp @@ -32,24 +32,24 @@ void interleave_block<1, 1, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x21, %x[width]\n" - "mov x20, %x[width]\n" - "incw x21\n" + "mov x22, %x[width]\n" + "incw x22\n" "cntw x10\n" - "sub x21, x21, #0x1\n" + "sub x22, x22, #0x1\n" + "udiv x22, x22, x10\n" // n_passes = ceildiv(width, VL<T>) + "mov x21, %x[width]\n" "sub x9, x10, #0x1\n" - "udiv x21, x21, x10\n" // n_passes = ceildiv(width, VL<T>) - "ands x9, x20, x9\n" - "sub x20, x21, #0x1\n" + "sub x20, x22, #0x1\n" + "ands x9, x21, x9\n" "sub x28, x10, #0x2\n" "mov x27, #0x0\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) - "ldr x24, [x26, #0x0]\n" + "and x24, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x23, [x26, #0x8]\n" "csel x9, x9, x10, NE\n" "ptrue p11.s\n" - "ldr x23, [x26, #0x8]\n" "whilelt p10.s, XZR, %x[height]\n" "mov x22, %x[row_offset]\n" "mov x21, %x[out]\n" @@ -59,119 +59,119 @@ void interleave_block<1, 1, VLType::SME, false>( "mov x12, #0x0\n" "cbz x28, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25306141 // psel p1.s, p8.s/Z, p10.s[w12]\n" + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" ".inst 0x25706140 // psel p0.s, p8.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0960700 // ld1w { za0h.s[x12] }, p1/Z, [x24, x22, LSL #2]\n" - "ldr x24, [x26, #0x0]\n" + "ldr x25, [x26, #0x0]\n" ".inst 0xe09602e1 // ld1w { za0h.s[x12, #1] }, p0/Z, [x23, x22, LSL #2]\n" "add x12, x12, #0x2\n" + "cmp x12, x28\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "cmp x12, x28\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25306141 // psel p1.s, p8.s/Z, p10.s[w12]\n" + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" ".inst 0x25706140 // psel p0.s, p8.s/Z, p10.s[w12, #1]\n" "mov x26, %x[in]\n" - "incw x27\n" - ".inst 0xe0960700 // ld1w { za0h.s[x12] }, p1/Z, [x24, x22, LSL #2]\n" - "ldr x24, [x26, #0x0]\n" ".inst 0xe09602e1 // ld1w { za0h.s[x12, #1] }, p0/Z, [x23, x22, LSL #2]\n" + "ldr x25, [x26, #0x0]\n" + "incw x22\n" "ldr x23, [x26, #0x8]\n" "add x26, x26, #0x10\n" - "incw x22\n" + "incw x27\n" "cbz x20, 8f\n" "mov x20, x20\n" "3:" // K loop: Main loop "whilelt p8.s, x27, %x[width]\n" - "mov x13, #0x0\n" + "mov x12, #0x0\n" "cbz x28, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25316143 // psel p3.s, p8.s/Z, p10.s[w13]\n" - ".inst 0x25716142 // psel p2.s, p8.s/Z, p10.s[w13, #1]\n" - ".inst 0x25316d21 // psel p1.s, p11.s/Z, p9.s[w13]\n" - ".inst 0x25716d20 // psel p0.s, p11.s/Z, p9.s[w13, #1]\n" - ".inst 0xe0962f08 // ld1w { za2h.s[x13] }, p3/Z, [x24, x22, LSL #2]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0962ae9 // ld1w { za2h.s[x13, #1] }, p2/Z, [x23, x22, LSL #2]\n" + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960328 // ld1w { za2h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0x25706141 // psel p1.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe09606e9 // ld1w { za2h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" "ldr x23, [x26, #0x8]\n" - ".inst 0xe0bfa6a0 // st1w { za0v.s[x13] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0aa82a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" "add x26, x26, #0x10\n" - ".inst 0xe0aaa2a1 // st1w { za0v.s[x13, #1] }, p0/Z, [x21, x10, LSL #2]\n" - "add x13, x13, #0x2\n" "addvl x21, x21, #2\n" - "cmp x13, x28\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25316143 // psel p3.s, p8.s/Z, p10.s[w13]\n" - ".inst 0x25716142 // psel p2.s, p8.s/Z, p10.s[w13, #1]\n" - ".inst 0x25316d21 // psel p1.s, p11.s/Z, p9.s[w13]\n" - ".inst 0x25716d20 // psel p0.s, p11.s/Z, p9.s[w13, #1]\n" + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960328 // ld1w { za2h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25706141 // psel p1.s, p8.s/Z, p10.s[w12, #1]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe09606e9 // ld1w { za2h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "whilelt p9.s, x27, %x[width]\n" - ".inst 0xe0962f08 // ld1w { za2h.s[x13] }, p3/Z, [x24, x22, LSL #2]\n" - "ldr x24, [x26, #0x0]\n" "incw x27\n" - "mov x12, #0x0\n" - ".inst 0xe0962ae9 // ld1w { za2h.s[x13, #1] }, p2/Z, [x23, x22, LSL #2]\n" - "ldr x23, [x26, #0x8]\n" - ".inst 0xe0bfa6a0 // st1w { za0v.s[x13] }, p1/Z, [x21, XZR, LSL #2]\n" "add x26, x26, #0x10\n" - ".inst 0xe0aaa2a1 // st1w { za0v.s[x13, #1] }, p0/Z, [x21, x10, LSL #2]\n" + ".inst 0xe0aa82a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" "addvl x21, x21, #2\n" "incw x22\n" "whilelt p8.s, x27, %x[width]\n" + "mov x12, #0x0\n" "cbz x28, 7f\n" "6:" // K loop: Main loop: Second: Loop - ".inst 0x25306143 // psel p3.s, p8.s/Z, p10.s[w12]\n" - ".inst 0x25706142 // psel p2.s, p8.s/Z, p10.s[w12, #1]\n" - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" - ".inst 0xe0960f00 // ld1w { za0h.s[x12] }, p3/Z, [x24, x22, LSL #2]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe0960ae1 // ld1w { za0h.s[x12, #1] }, p2/Z, [x23, x22, LSL #2]\n" + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0x25706141 // psel p1.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe09606e1 // ld1w { za0h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" "ldr x23, [x26, #0x8]\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" - "add x26, x26, #0x10\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" "add x12, x12, #0x2\n" - "addvl x21, x21, #2\n" "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail - ".inst 0x25306143 // psel p3.s, p8.s/Z, p10.s[w12]\n" - ".inst 0x25706142 // psel p2.s, p8.s/Z, p10.s[w12, #1]\n" - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25706141 // psel p1.s, p8.s/Z, p10.s[w12, #1]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe09606e1 // ld1w { za0h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "whilelt p9.s, x27, %x[width]\n" - ".inst 0xe0960f00 // ld1w { za0h.s[x12] }, p3/Z, [x24, x22, LSL #2]\n" - "ldr x24, [x26, #0x0]\n" "subs x20, x20, #0x1\n" - "incw x27\n" - ".inst 0xe0960ae1 // ld1w { za0h.s[x12, #1] }, p2/Z, [x23, x22, LSL #2]\n" - "ldr x23, [x26, #0x8]\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" "add x26, x26, #0x10\n" ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" "addvl x21, x21, #2\n" + "incw x27\n" "incw x22\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x25, 11f\n" + "cbnz x24, 11f\n" "mov x26, %x[in]\n" "whilelt p8.s, x27, %x[width]\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" - ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" - "addvl x21, x21, #1\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "ldr x20, [x26, #0x0]\n" - "add x26, x26, #0x8\n" + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" ".inst 0xe0960288 // ld1w { za2h.s[x12] }, p0/Z, [x20, x22, LSL #2]\n" "add x12, x12, #0x1\n" "cmp x12, x10\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" "blt 9b\n" "whilelt p9.s, x27, %x[width]\n" "whilelt p8.s, x27, %x[width]\n" @@ -180,8 +180,8 @@ void interleave_block<1, 1, VLType::SME, false>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x9\n" + "addvl x21, x21, #1\n" "blt 10b\n" "whilelt p8.s, x27, %x[width]\n" "b 13f\n" @@ -191,15 +191,15 @@ void interleave_block<1, 1, VLType::SME, false>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x21, x21, #1\n" "cmp x12, x9\n" + "addvl x21, x21, #1\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x21\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp index a07831e7bd..1ed835b21b 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp @@ -32,66 +32,66 @@ void interleave_block<2, 1, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x28, #0x0\n" - "mov x27, %x[row_offset]\n" - "cnth x26\n" - "cnth x25\n" - "cmp %x[height], x26\n" + "cnth x28\n" + "cmp %x[height], x28\n" + "cnth x27\n" + "csel x28, %x[height], x28, LT\n" + "mov x26, #0x0\n" "ptrue p13.s\n" - "csel x26, %x[height], x26, LT\n" + "sub x28, x28, #0x1\n" "whilelt p12.h, XZR, %x[height]\n" - "sub x26, x26, #0x1\n" - "whilelt p11.h, x25, %x[height]\n" + "whilelt p11.h, x27, %x[height]\n" + "mov x25, %x[row_offset]\n" "mov x24, %x[out]\n" - "whilelt p10.h, x28, %x[width]\n" - "whilelt p9.h, x28, %x[width]\n" - "whilelt p8.h, x28, %x[width]\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" "1:" // Width loop "add x23, %x[in], XZR, LSL #3\n" - "add x20, %x[in], x25, LSL #3\n" - "mov x13, #0x0\n" + "add x20, %x[in], x27, LSL #3\n" "ldr x22, [x23], #0x8\n" + "mov x12, #0x0\n" "ldr x21, [x20], #0x8\n" - "cbz x26, 3f\n" + "cbz x28, 3f\n" "2:" // Loads: Loop - ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0xe05b26c0 // ld1h { za0h.h[x13] }, p1/Z, [x22, x27, LSL #1]\n" + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05906c0 // ld1h { za0h.h[x12] }, p1/Z, [x22, x25, LSL #1]\n" "ldr x22, [x23], #0x8\n" - ".inst 0xe05b22a8 // ld1h { za1h.h[x13] }, p0/Z, [x21, x27, LSL #1]\n" - "add x13, x13, #0x2\n" + ".inst 0xe05902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28, LSL #1\n" "ldr x21, [x20], #0x8\n" - "cmp x13, x26, LSL #1\n" "blt 2b\n" "3:" // Loads: Tail - ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - "sub x20, %x[width], x28\n" + "sub x20, %x[width], x26\n" + ".inst 0x25286580 // psel p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05902c0 // ld1h { za0h.h[x12] }, p0/Z, [x22, x25, LSL #1]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + "cmp x20, x27\n" + ".inst 0xe05902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" "mov x12, #0x0\n" - "cmp x20, x25\n" - ".inst 0xe05b26c0 // ld1h { za0h.h[x13] }, p1/Z, [x22, x27, LSL #1]\n" - "csel x20, x20, x25, LT\n" - ".inst 0xe05b22a8 // ld1h { za1h.h[x13] }, p0/Z, [x21, x27, LSL #1]\n" + "csel x20, x20, x27, LT\n" "4:" // Stores: Loop - ".inst 0x25287541 // psel p1.h, p13.h/Z, p10.h[w12]\n" ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" - ".inst 0xe07f8700 // st1h { za0v.h[x12] }, p1/Z, [x24, XZR, LSL #1]\n" - ".inst 0xe0798308 // st1h { za1v.h[x12] }, p0/Z, [x24, x25, LSL #1]\n" + ".inst 0xe07f8300 // st1h { za0v.h[x12] }, p0/Z, [x24, XZR, LSL #1]\n" + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07b8308 // st1h { za1v.h[x12] }, p0/Z, [x24, x27, LSL #1]\n" "add x12, x12, #0x1\n" - "addvl x24, x24, #4\n" "cmp x12, x20\n" + "addvl x24, x24, #4\n" "blt 4b\n" - "inch x28\n" - "inch x27\n" - "whilelt p10.h, x28, %x[width]\n" - "whilelt p9.h, x28, %x[width]\n" - "whilelt p8.h, x28, %x[width]\n" + "inch x26\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "inch x25\n" "b.any 1b\n" "mov %x[out], x24\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp index 01dfecc4ef..715810ddea 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp @@ -32,264 +32,269 @@ void interleave_block<2, 2, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x22, %x[width]\n" + "cnth x22\n" "mov x21, %x[width]\n" - "cnth x20\n" - "inch x22\n" - "sub x7, x20, #0x1\n" - "sub x22, x22, #0x1\n" - "ands x7, x21, x7\n" - "cntw x8\n" - "udiv x22, x22, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x7, x7, x20, NE\n" - "sub x13, x22, #0x1\n" - "add x7, x7, #0x1\n" - "sub x17, x8, #0x2\n" - "lsl x21, %x[height], #0x1\n" // height * 2 - "lsl x20, x8, #0x1\n" - "mov x16, #0x0\n" + "inch x21\n" + "mov x20, %x[width]\n" + "sub x17, x22, #0x1\n" + "sub x21, x21, #0x1\n" + "ands x17, x20, x17\n" + "cntw x16\n" + "udiv x21, x21, x22\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x22, NE\n" + "sub x13, x21, #0x1\n" + "add x17, x17, #0x1\n" + "sub x15, x16, #0x2\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x20, x16, #0x1\n" + "mov x14, #0x0\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - "cntw x9, ALL, MUL #2\n" - "cntw x28, ALL, MUL #3\n" - "ldr x27, [x11, #0x0]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + "cntw x28, ALL, MUL #2\n" + "cntw x27, ALL, MUL #3\n" + "ldr x26, [x10, #0x0]\n" "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x26, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) - "ldr x25, [x10, #0x0]\n" - "lsr x7, x7, #0x1\n" - "ptrue p12.s\n" + "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) "ldr x24, [x11, #0x8]\n" - "whilelt p11.h, XZR, x21\n" - "whilelt p10.h, x20, x21\n" + "lsr x17, x17, #0x1\n" + "ptrue p13.s\n" "ldr x21, [x10, #0x8]\n" + "whilelt p12.h, XZR, x22\n" + "whilelt p11.h, x20, x22\n" "mov x23, %x[row_offset]\n" "mov x22, %x[out]\n" - "whilelt p9.h, x16, %x[width]\n" - "whilelt p8.h, x16, %x[width]\n" + "whilelt p10.h, x14, %x[width]\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" "mov x12, #0x0\n" - "cbz x17, 2f\n" + "cbz x15, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25286163 // psel p3.h, p8.h/Z, p11.h[w12]\n" - ".inst 0x25286142 // psel p2.h, p8.h/Z, p10.h[w12]\n" - ".inst 0x25686161 // psel p1.h, p8.h/Z, p11.h[w12, #2]\n" - ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" - ".inst 0xe0570f60 // ld1h { za0h.h[x12] }, p3/Z, [x27, x23, LSL #1]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0570b28 // ld1h { za1h.h[x12] }, p2/Z, [x25, x23, LSL #1]\n" - "ldr x25, [x10, #0x0]\n" + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" + "ldr x26, [x10, #0x0]\n" ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" "add x12, x12, #0x4\n" + "cmp x12, x15, LSL #1\n" "ldr x21, [x10, #0x8]\n" "add x10, x10, #0x10\n" - "cmp x12, x17, LSL #1\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25286163 // psel p3.h, p8.h/Z, p11.h[w12]\n" - ".inst 0x25286142 // psel p2.h, p8.h/Z, p10.h[w12]\n" - ".inst 0x25686161 // psel p1.h, p8.h/Z, p11.h[w12, #2]\n" - ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0570f60 // ld1h { za0h.h[x12] }, p3/Z, [x27, x23, LSL #1]\n" - "ldr x27, [x11, #0x0]\n" - "inch x16\n" - ".inst 0xe0570b28 // ld1h { za1h.h[x12] }, p2/Z, [x25, x23, LSL #1]\n" - "ldr x25, [x10, #0x0]\n" + "add x10, %x[in], x16, LSL #3\n" ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + "inch x23\n" + "inch x14\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" "ldr x21, [x10, #0x8]\n" "add x10, x10, #0x10\n" - "inch x23\n" "cbz x13, 8f\n" "mov x20, x13\n" "3:" // K loop: Main loop - "whilelt p8.h, x16, %x[width]\n" - "mov x15, #0x0\n" - "mov x14, #0x0\n" - "cbz x17, 5f\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x253b6160 // psel p0.h, p8.h/Z, p11.h[w15, #1]\n" - ".inst 0x253b6142 // psel p2.h, p8.h/Z, p10.h[w15, #1]\n" - ".inst 0x257b6161 // psel p1.h, p8.h/Z, p11.h[w15, #3]\n" - ".inst 0x257b6143 // psel p3.h, p8.h/Z, p10.h[w15, #3]\n" - ".inst 0xe0576361 // ld1h { za0h.h[x15, #1] }, p0/Z, [x27, x23, LSL #1]\n" - ".inst 0x252a7120 // psel p0.h, p12.h/Z, p9.h[w14]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0576b29 // ld1h { za1h.h[x15, #1] }, p2/Z, [x25, x23, LSL #1]\n" - ".inst 0x252a7122 // psel p2.h, p12.h/Z, p9.h[w14]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0576703 // ld1h { za0h.h[x15, #3] }, p1/Z, [x24, x23, LSL #1]\n" - ".inst 0x253a7121 // psel p1.h, p12.h/Z, p9.h[w14, #1]\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0x25796162 // psel p2.h, p8.h/Z, p11.h[w13, #3]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe0576eab // ld1h { za1h.h[x15, #3] }, p3/Z, [x21, x23, LSL #1]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572aab // ld1h { za1h.h[x13, #3] }, p2/Z, [x21, x23, LSL #1]\n" "ldr x21, [x10, #0x8]\n" - ".inst 0xe0bfc2c0 // st1w { za0v.s[x14] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x253a7120 // psel p0.h, p12.h/Z, p9.h[w14, #1]\n" - ".inst 0xe0a8cac4 // st1w { za1v.s[x14] }, p2/Z, [x22, x8, LSL #2]\n" + ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x10, x10, #0x10\n" - "add x15, x15, #0x4\n" - ".inst 0xe0a9c6c1 // st1w { za0v.s[x14, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bcc2c5 // st1w { za1v.s[x14, #1] }, p0/Z, [x22, x28, LSL #2]\n" - "add x14, x14, #0x2\n" + "add x13, x13, #0x4\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" "addvl x22, x22, #4\n" - "cmp x14, x17\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x253b6160 // psel p0.h, p8.h/Z, p11.h[w15, #1]\n" - ".inst 0x253b6142 // psel p2.h, p8.h/Z, p10.h[w15, #1]\n" - ".inst 0x257b6161 // psel p1.h, p8.h/Z, p11.h[w15, #3]\n" - ".inst 0x257b6143 // psel p3.h, p8.h/Z, p10.h[w15, #3]\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0576361 // ld1h { za0h.h[x15, #1] }, p0/Z, [x27, x23, LSL #1]\n" - ".inst 0x252a7120 // psel p0.h, p12.h/Z, p9.h[w14]\n" - "ldr x27, [x11, #0x0]\n" - "mov x13, #0x0\n" - ".inst 0xe0576b29 // ld1h { za1h.h[x15, #1] }, p2/Z, [x25, x23, LSL #1]\n" - ".inst 0x252a7122 // psel p2.h, p12.h/Z, p9.h[w14]\n" - "ldr x25, [x10, #0x0]\n" - "mov x12, #0x0\n" - ".inst 0xe0576703 // ld1h { za0h.h[x15, #3] }, p1/Z, [x24, x23, LSL #1]\n" - ".inst 0x253a7121 // psel p1.h, p12.h/Z, p9.h[w14, #1]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0x25796161 // psel p1.h, p8.h/Z, p11.h[w13, #3]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe05726ab // ld1h { za1h.h[x13, #3] }, p1/Z, [x21, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe0576eab // ld1h { za1h.h[x15, #3] }, p3/Z, [x21, x23, LSL #1]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" "ldr x21, [x10, #0x8]\n" - ".inst 0xe0bfc2c0 // st1w { za0v.s[x14] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x253a7120 // psel p0.h, p12.h/Z, p9.h[w14, #1]\n" - ".inst 0xe0a8cac4 // st1w { za1v.s[x14] }, p2/Z, [x22, x8, LSL #2]\n" - "whilelt p9.h, x16, %x[width]\n" - "inch x16\n" - ".inst 0xe0a9c6c1 // st1w { za0v.s[x14, #1] }, p1/Z, [x22, x9, LSL #2]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.h, x14, %x[width]\n" + "inch x14\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - "inch x23\n" - ".inst 0xe0bcc2c5 // st1w { za1v.s[x14, #1] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "addvl x22, x22, #4\n" - "whilelt p8.h, x16, %x[width]\n" - "cbz x17, 7f\n" + "inch x23\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 7f\n" "6:" // K loop: Main loop: Second: Loop + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0x25296142 // psel p2.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" - ".inst 0x25696143 // psel p3.h, p8.h/Z, p10.h[w13, #2]\n" - ".inst 0xe0572360 // ld1h { za0h.h[x13] }, p0/Z, [x27, x23, LSL #1]\n" - ".inst 0x25287120 // psel p0.h, p12.h/Z, p9.h[w12]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0572b28 // ld1h { za1h.h[x13] }, p2/Z, [x25, x23, LSL #1]\n" - ".inst 0x25287122 // psel p2.h, p12.h/Z, p9.h[w12]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0572702 // ld1h { za0h.h[x13, #2] }, p1/Z, [x24, x23, LSL #1]\n" - ".inst 0x25387121 // psel p1.h, p12.h/Z, p9.h[w12, #1]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0x25696162 // psel p2.h, p8.h/Z, p11.h[w13, #2]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe0572eaa // ld1h { za1h.h[x13, #2] }, p3/Z, [x21, x23, LSL #1]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572aaa // ld1h { za1h.h[x13, #2] }, p2/Z, [x21, x23, LSL #1]\n" "ldr x21, [x10, #0x8]\n" - ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x25387120 // psel p0.h, p12.h/Z, p9.h[w12, #1]\n" - ".inst 0xe0a88acc // st1w { za3v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" + ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x10, x10, #0x10\n" "add x13, x13, #0x4\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bc82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "add x12, x12, #0x2\n" + "cmp x12, x15\n" "addvl x22, x22, #4\n" - "cmp x12, x17\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0x25296142 // psel p2.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" - ".inst 0x25696143 // psel p3.h, p8.h/Z, p10.h[w13, #2]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0572360 // ld1h { za0h.h[x13] }, p0/Z, [x27, x23, LSL #1]\n" - ".inst 0x25287120 // psel p0.h, p12.h/Z, p9.h[w12]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0572b28 // ld1h { za1h.h[x13] }, p2/Z, [x25, x23, LSL #1]\n" - ".inst 0x25287122 // psel p2.h, p12.h/Z, p9.h[w12]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0572702 // ld1h { za0h.h[x13, #2] }, p1/Z, [x24, x23, LSL #1]\n" - ".inst 0x25387121 // psel p1.h, p12.h/Z, p9.h[w12, #1]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe05726aa // ld1h { za1h.h[x13, #2] }, p1/Z, [x21, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe0572eaa // ld1h { za1h.h[x13, #2] }, p3/Z, [x21, x23, LSL #1]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" "ldr x21, [x10, #0x8]\n" ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x25387120 // psel p0.h, p12.h/Z, p9.h[w12, #1]\n" - ".inst 0xe0a88acc // st1w { za3v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" - "whilelt p9.h, x16, %x[width]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08acc // st1w { za3v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.h, x14, %x[width]\n" "subs x20, x20, #0x1\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - "inch x16\n" - ".inst 0xe0bc82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "addvl x22, x22, #4\n" + "inch x14\n" "inch x23\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x26, 11f\n" + "cbnz x25, 11f\n" "mov x11, %x[in]\n" - "whilelt p8.h, x16, %x[width]\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "mov x13, #0x0\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25307123 // psel p3.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25307122 // psel p2.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25396161 // psel p1.h, p8.h/Z, p11.h[w13, #1]\n" - ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" - ".inst 0xe0bf8ec0 // st1w { za0v.s[x12] }, p3/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a88ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" - "add x12, x12, #0x1\n" - "addvl x22, x22, #2\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "ldr x21, [x11, #0x0]\n" - "cmp x12, x8\n" - "ldr x20, [x11, x8, LSL #0x3]\n" - "add x11, x11, #0x8\n" + "add x12, x12, #0x1\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + "cmp x12, x16\n" ".inst 0xe05726a1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x21, x23, LSL #1]\n" ".inst 0xe0572289 // ld1h { za1h.h[x13, #1] }, p0/Z, [x20, x23, LSL #1]\n" + "add x11, x11, #0x8\n" + "addvl x22, x22, #2\n" "add x13, x13, #0x2\n" "blt 9b\n" - "whilelt p9.h, x16, %x[width]\n" - "whilelt p8.h, x16, %x[width]\n" + "whilelt p10.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "mov x20, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second - ".inst 0x25307121 // psel p1.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25307120 // psel p0.s, p12.s/Z, p9.s[w12]\n" - "add x20, x20, #0x2\n" - ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a882cc // st1w { za3v.s[x12] }, p0/Z, [x22, x8, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" + "cmp x12, x17\n" "addvl x22, x22, #2\n" - "cmp x12, x7\n" + "add x20, x20, #0x2\n" "blt 10b\n" - "whilelt p8.h, x16, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" "12:" // K loop: Tails: Odd: Loop - ".inst 0x25307121 // psel p1.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25307120 // psel p0.s, p12.s/Z, p9.s[w12]\n" - ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a882c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x8, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" + "cmp x12, x17\n" "addvl x22, x22, #2\n" - "cmp x12, x7\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x22\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp index b1e9226773..849f6c3228 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp @@ -32,264 +32,269 @@ void interleave_block<2, 2, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x22, %x[width]\n" + "cnth x22\n" "mov x21, %x[width]\n" - "cnth x20\n" - "inch x22\n" - "sub x7, x20, #0x1\n" - "sub x22, x22, #0x1\n" - "ands x7, x21, x7\n" - "cntw x8\n" - "udiv x22, x22, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x7, x7, x20, NE\n" - "sub x13, x22, #0x1\n" - "add x7, x7, #0x1\n" - "sub x17, x8, #0x2\n" - "lsl x21, %x[height], #0x1\n" // height * 2 - "lsl x20, x8, #0x1\n" - "mov x16, #0x0\n" + "inch x21\n" + "mov x20, %x[width]\n" + "sub x17, x22, #0x1\n" + "sub x21, x21, #0x1\n" + "ands x17, x20, x17\n" + "cntw x16\n" + "udiv x21, x21, x22\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x22, NE\n" + "sub x13, x21, #0x1\n" + "add x17, x17, #0x1\n" + "sub x15, x16, #0x2\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x20, x16, #0x1\n" + "mov x14, #0x0\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - "cntw x9, ALL, MUL #2\n" - "cntw x28, ALL, MUL #3\n" - "ldr x27, [x11, #0x0]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + "cntw x28, ALL, MUL #2\n" + "cntw x27, ALL, MUL #3\n" + "ldr x26, [x10, #0x0]\n" "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x26, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) - "ldr x25, [x10, #0x0]\n" - "lsr x7, x7, #0x1\n" - "ptrue p12.s\n" + "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) "ldr x24, [x11, #0x8]\n" - "whilelt p11.h, XZR, x21\n" - "whilelt p10.h, x20, x21\n" + "lsr x17, x17, #0x1\n" + "ptrue p13.s\n" "ldr x21, [x10, #0x8]\n" + "whilelt p12.h, XZR, x22\n" + "whilelt p11.h, x20, x22\n" "mov x23, %x[row_offset]\n" "mov x22, %x[out]\n" - "whilelt p9.h, x16, %x[width]\n" - "whilelt p8.h, x16, %x[width]\n" + "whilelt p10.h, x14, %x[width]\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" "mov x12, #0x0\n" - "cbz x17, 2f\n" + "cbz x15, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25286163 // psel p3.h, p8.h/Z, p11.h[w12]\n" - ".inst 0x25286142 // psel p2.h, p8.h/Z, p10.h[w12]\n" - ".inst 0x25686161 // psel p1.h, p8.h/Z, p11.h[w12, #2]\n" - ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" - ".inst 0xe0570f60 // ld1h { za0h.h[x12] }, p3/Z, [x27, x23, LSL #1]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0570b28 // ld1h { za1h.h[x12] }, p2/Z, [x25, x23, LSL #1]\n" - "ldr x25, [x10, #0x0]\n" + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" + "ldr x26, [x10, #0x0]\n" ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" "add x12, x12, #0x4\n" + "cmp x12, x15, LSL #1\n" "ldr x21, [x10, #0x8]\n" "add x10, x10, #0x10\n" - "cmp x12, x17, LSL #1\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25286163 // psel p3.h, p8.h/Z, p11.h[w12]\n" - ".inst 0x25286142 // psel p2.h, p8.h/Z, p10.h[w12]\n" - ".inst 0x25686161 // psel p1.h, p8.h/Z, p11.h[w12, #2]\n" - ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0570f60 // ld1h { za0h.h[x12] }, p3/Z, [x27, x23, LSL #1]\n" - "ldr x27, [x11, #0x0]\n" - "inch x16\n" - ".inst 0xe0570b28 // ld1h { za1h.h[x12] }, p2/Z, [x25, x23, LSL #1]\n" - "ldr x25, [x10, #0x0]\n" + "add x10, %x[in], x16, LSL #3\n" ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + "inch x23\n" + "inch x14\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" "ldr x21, [x10, #0x8]\n" "add x10, x10, #0x10\n" - "inch x23\n" "cbz x13, 8f\n" "mov x20, x13\n" "3:" // K loop: Main loop - "whilelt p8.h, x16, %x[width]\n" - "mov x15, #0x0\n" - "mov x14, #0x0\n" - "cbz x17, 5f\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x253b6160 // psel p0.h, p8.h/Z, p11.h[w15, #1]\n" - ".inst 0x253b6142 // psel p2.h, p8.h/Z, p10.h[w15, #1]\n" - ".inst 0x257b6161 // psel p1.h, p8.h/Z, p11.h[w15, #3]\n" - ".inst 0x257b6143 // psel p3.h, p8.h/Z, p10.h[w15, #3]\n" - ".inst 0xe0576361 // ld1h { za0h.h[x15, #1] }, p0/Z, [x27, x23, LSL #1]\n" - ".inst 0x252a7120 // psel p0.h, p12.h/Z, p9.h[w14]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0576b29 // ld1h { za1h.h[x15, #1] }, p2/Z, [x25, x23, LSL #1]\n" - ".inst 0x252a7122 // psel p2.h, p12.h/Z, p9.h[w14]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0576703 // ld1h { za0h.h[x15, #3] }, p1/Z, [x24, x23, LSL #1]\n" - ".inst 0x253a7121 // psel p1.h, p12.h/Z, p9.h[w14, #1]\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0x25796162 // psel p2.h, p8.h/Z, p11.h[w13, #3]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe0576eab // ld1h { za1h.h[x15, #3] }, p3/Z, [x21, x23, LSL #1]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572aab // ld1h { za1h.h[x13, #3] }, p2/Z, [x21, x23, LSL #1]\n" "ldr x21, [x10, #0x8]\n" - ".inst 0xe0bfc2c0 // st1w { za0v.s[x14] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x253a7120 // psel p0.h, p12.h/Z, p9.h[w14, #1]\n" - ".inst 0xe0a8cac4 // st1w { za1v.s[x14] }, p2/Z, [x22, x8, LSL #2]\n" + ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x10, x10, #0x10\n" - "add x15, x15, #0x4\n" - ".inst 0xe0a9c6c1 // st1w { za0v.s[x14, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bcc2c5 // st1w { za1v.s[x14, #1] }, p0/Z, [x22, x28, LSL #2]\n" - "add x14, x14, #0x2\n" + "add x13, x13, #0x4\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" "addvl x22, x22, #4\n" - "cmp x14, x17\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x253b6160 // psel p0.h, p8.h/Z, p11.h[w15, #1]\n" - ".inst 0x253b6142 // psel p2.h, p8.h/Z, p10.h[w15, #1]\n" - ".inst 0x257b6161 // psel p1.h, p8.h/Z, p11.h[w15, #3]\n" - ".inst 0x257b6143 // psel p3.h, p8.h/Z, p10.h[w15, #3]\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0576361 // ld1h { za0h.h[x15, #1] }, p0/Z, [x27, x23, LSL #1]\n" - ".inst 0x252a7120 // psel p0.h, p12.h/Z, p9.h[w14]\n" - "ldr x27, [x11, #0x0]\n" - "mov x13, #0x0\n" - ".inst 0xe0576b29 // ld1h { za1h.h[x15, #1] }, p2/Z, [x25, x23, LSL #1]\n" - ".inst 0x252a7122 // psel p2.h, p12.h/Z, p9.h[w14]\n" - "ldr x25, [x10, #0x0]\n" - "mov x12, #0x0\n" - ".inst 0xe0576703 // ld1h { za0h.h[x15, #3] }, p1/Z, [x24, x23, LSL #1]\n" - ".inst 0x253a7121 // psel p1.h, p12.h/Z, p9.h[w14, #1]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0x25796161 // psel p1.h, p8.h/Z, p11.h[w13, #3]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe05726ab // ld1h { za1h.h[x13, #3] }, p1/Z, [x21, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe0576eab // ld1h { za1h.h[x15, #3] }, p3/Z, [x21, x23, LSL #1]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" "ldr x21, [x10, #0x8]\n" - ".inst 0xe0bfc2c0 // st1w { za0v.s[x14] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x253a7120 // psel p0.h, p12.h/Z, p9.h[w14, #1]\n" - ".inst 0xe0a8cac4 // st1w { za1v.s[x14] }, p2/Z, [x22, x8, LSL #2]\n" - "whilelt p9.h, x16, %x[width]\n" - "inch x16\n" - ".inst 0xe0a9c6c1 // st1w { za0v.s[x14, #1] }, p1/Z, [x22, x9, LSL #2]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.h, x14, %x[width]\n" + "inch x14\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - "inch x23\n" - ".inst 0xe0bcc2c5 // st1w { za1v.s[x14, #1] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "addvl x22, x22, #4\n" - "whilelt p8.h, x16, %x[width]\n" - "cbz x17, 7f\n" + "inch x23\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 7f\n" "6:" // K loop: Main loop: Second: Loop + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0x25296142 // psel p2.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" - ".inst 0x25696143 // psel p3.h, p8.h/Z, p10.h[w13, #2]\n" - ".inst 0xe0572360 // ld1h { za0h.h[x13] }, p0/Z, [x27, x23, LSL #1]\n" - ".inst 0x25287120 // psel p0.h, p12.h/Z, p9.h[w12]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0572b28 // ld1h { za1h.h[x13] }, p2/Z, [x25, x23, LSL #1]\n" - ".inst 0x25287122 // psel p2.h, p12.h/Z, p9.h[w12]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0572702 // ld1h { za0h.h[x13, #2] }, p1/Z, [x24, x23, LSL #1]\n" - ".inst 0x25387121 // psel p1.h, p12.h/Z, p9.h[w12, #1]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0x25696162 // psel p2.h, p8.h/Z, p11.h[w13, #2]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe0572eaa // ld1h { za1h.h[x13, #2] }, p3/Z, [x21, x23, LSL #1]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572aaa // ld1h { za1h.h[x13, #2] }, p2/Z, [x21, x23, LSL #1]\n" "ldr x21, [x10, #0x8]\n" - ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x25387120 // psel p0.h, p12.h/Z, p9.h[w12, #1]\n" - ".inst 0xe0a88acc // st1w { za3v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" + ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x10, x10, #0x10\n" "add x13, x13, #0x4\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bc82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "add x12, x12, #0x2\n" + "cmp x12, x15\n" "addvl x22, x22, #4\n" - "cmp x12, x17\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0x25296142 // psel p2.h, p8.h/Z, p10.h[w13]\n" - ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" - ".inst 0x25696143 // psel p3.h, p8.h/Z, p10.h[w13, #2]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0572360 // ld1h { za0h.h[x13] }, p0/Z, [x27, x23, LSL #1]\n" - ".inst 0x25287120 // psel p0.h, p12.h/Z, p9.h[w12]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0572b28 // ld1h { za1h.h[x13] }, p2/Z, [x25, x23, LSL #1]\n" - ".inst 0x25287122 // psel p2.h, p12.h/Z, p9.h[w12]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0572702 // ld1h { za0h.h[x13, #2] }, p1/Z, [x24, x23, LSL #1]\n" - ".inst 0x25387121 // psel p1.h, p12.h/Z, p9.h[w12, #1]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe05726aa // ld1h { za1h.h[x13, #2] }, p1/Z, [x21, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe0572eaa // ld1h { za1h.h[x13, #2] }, p3/Z, [x21, x23, LSL #1]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" "ldr x21, [x10, #0x8]\n" ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x25387120 // psel p0.h, p12.h/Z, p9.h[w12, #1]\n" - ".inst 0xe0a88acc // st1w { za3v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" - "whilelt p9.h, x16, %x[width]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08acc // st1w { za3v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.h, x14, %x[width]\n" "subs x20, x20, #0x1\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - "inch x16\n" - ".inst 0xe0bc82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "addvl x22, x22, #4\n" + "inch x14\n" "inch x23\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x26, 11f\n" + "cbnz x25, 11f\n" "mov x11, %x[in]\n" - "whilelt p8.h, x16, %x[width]\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "mov x13, #0x0\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25307123 // psel p3.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25307122 // psel p2.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25396161 // psel p1.h, p8.h/Z, p11.h[w13, #1]\n" - ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" - ".inst 0xe0bf8ec0 // st1w { za0v.s[x12] }, p3/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a88ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" - "add x12, x12, #0x1\n" - "addvl x22, x22, #2\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "ldr x21, [x11, #0x0]\n" - "cmp x12, x8\n" - "ldr x20, [x11, x8, LSL #0x3]\n" - "add x11, x11, #0x8\n" + "add x12, x12, #0x1\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + "cmp x12, x16\n" ".inst 0xe05726a1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x21, x23, LSL #1]\n" ".inst 0xe0572289 // ld1h { za1h.h[x13, #1] }, p0/Z, [x20, x23, LSL #1]\n" + "add x11, x11, #0x8\n" + "addvl x22, x22, #2\n" "add x13, x13, #0x2\n" "blt 9b\n" - "whilelt p9.h, x16, %x[width]\n" - "whilelt p8.h, x16, %x[width]\n" + "whilelt p10.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "mov x20, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second - ".inst 0x25307121 // psel p1.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25307120 // psel p0.s, p12.s/Z, p9.s[w12]\n" - "add x20, x20, #0x2\n" - ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a882cc // st1w { za3v.s[x12] }, p0/Z, [x22, x8, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" + "cmp x12, x17\n" "addvl x22, x22, #2\n" - "cmp x12, x7\n" + "add x20, x20, #0x2\n" "blt 10b\n" - "whilelt p8.h, x16, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" "12:" // K loop: Tails: Odd: Loop - ".inst 0x25307121 // psel p1.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25307120 // psel p0.s, p12.s/Z, p9.s[w12]\n" - ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a882c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x8, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" + "cmp x12, x17\n" "addvl x22, x22, #2\n" - "cmp x12, x7\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x22\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp index 7b42b6fb93..2d6e1ce6c7 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp @@ -32,265 +32,265 @@ void interleave_block<2, 4, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" "mov x23, %x[width]\n" - "mov x21, %x[width]\n" - "cntb x20\n" "incb x23\n" - "sub x7, x20, #0x1\n" - "cntw x8\n" + "mov x20, %x[width]\n" + "sub x17, x21, #0x1\n" + "cntw x16\n" "sub x23, x23, #0x1\n" - "ands x7, x21, x7\n" - "udiv x23, x23, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x7, x7, x20, NE\n" + "ands x17, x20, x17\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x21, NE\n" "lsl x22, %x[height], #0x1\n" // height * 2 - "lsl x21, x8, #0x1\n" + "lsl x21, x16, #0x1\n" "sub x20, x23, #0x1\n" - "add x7, x7, #0x3\n" - "sub x17, x8, #0x2\n" + "add x17, x17, #0x3\n" + "sub x15, x16, #0x2\n" "whilelt p9.b, XZR, x22\n" "whilelt p8.b, x21, x22\n" - "mov x16, #0x0\n" + "mov x14, #0x0\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - "cntw x9, ALL, MUL #2\n" - "cntw x28, ALL, MUL #3\n" - "ldr x27, [x11, #0x0]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + "cntw x28, ALL, MUL #2\n" + "cntw x27, ALL, MUL #3\n" + "ldr x26, [x10, #0x0]\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x26, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) - "ldr x25, [x10, #0x0]\n" - "lsr x7, x7, #0x2\n" - "ptrue p11.s\n" + "and x25, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) "ldr x24, [x11, #0x8]\n" + "lsr x17, x17, #0x2\n" + "ptrue p11.s\n" + "ldr x23, [x10, #0x8]\n" "zip1 p10.b, p9.b, p8.b\n" - "mov x23, %x[row_offset]\n" - "ldr x21, [x10, #0x8]\n" - "mov x22, %x[out]\n" - "whilelt p9.b, x16, %x[width]\n" - "whilelt p8.b, x16, %x[width]\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" "mov x12, #0x0\n" - "cbz x17, 2f\n" + "cbz x15, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25246143 // psel p3.b, p8.b/Z, p10.b[w12]\n" - ".inst 0x252c6142 // psel p2.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160120 // ld1b { za0h.b[x12] }, p0/Z, [x9, x22]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0160341 // ld1b { za0h.b[x12, #1] }, p0/Z, [x26, x22]\n" ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" - ".inst 0xe0170f60 // ld1b { za0h.b[x12] }, p3/Z, [x27, x23]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0170b21 // ld1b { za0h.b[x12, #1] }, p2/Z, [x25, x23]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0170704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x23]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0160704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x22]\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - ".inst 0xe01702a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x23]\n" + ".inst 0xe01602e5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x23, x22]\n" "add x12, x12, #0x8\n" - "ldr x21, [x10, #0x8]\n" + "cmp x12, x15, LSL #2\n" + "ldr x23, [x10, #0x8]\n" "add x10, x10, #0x10\n" - "cmp x12, x17, LSL #2\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25246143 // psel p3.b, p8.b/Z, p10.b[w12]\n" - ".inst 0x252c6142 // psel p2.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160120 // ld1b { za0h.b[x12] }, p0/Z, [x9, x22]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe0160341 // ld1b { za0h.b[x12, #1] }, p0/Z, [x26, x22]\n" ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + ".inst 0xe0160704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x22]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0170f60 // ld1b { za0h.b[x12] }, p3/Z, [x27, x23]\n" - "ldr x27, [x11, #0x0]\n" - "incb x16\n" - ".inst 0xe0170b21 // ld1b { za0h.b[x12, #1] }, p2/Z, [x25, x23]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0170704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x23]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe01602e5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x23, x22]\n" + "ldr x26, [x10, #0x0]\n" + "incb x22\n" + "incb x14\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - ".inst 0xe01702a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" + "ldr x23, [x10, #0x8]\n" "add x10, x10, #0x10\n" - "incb x23\n" "cbz x20, 8f\n" "mov x20, x20\n" "3:" // K loop: Main loop - "whilelt p8.b, x16, %x[width]\n" - "mov x15, #0x0\n" - "mov x14, #0x0\n" - "cbz x17, 5f\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25376143 // psel p3.b, p8.b/Z, p10.b[w15, #2]\n" - ".inst 0x253f6142 // psel p2.b, p8.b/Z, p10.b[w15, #3]\n" - ".inst 0x25776141 // psel p1.b, p8.b/Z, p10.b[w15, #6]\n" - ".inst 0x257f6140 // psel p0.b, p8.b/Z, p10.b[w15, #7]\n" - ".inst 0xe0176f62 // ld1b { za0h.b[x15, #2] }, p3/Z, [x27, x23]\n" - ".inst 0x25266d23 // psel p3.b, p11.b/Z, p9.b[w14]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0176b23 // ld1b { za0h.b[x15, #3] }, p2/Z, [x25, x23]\n" - ".inst 0x25266d22 // psel p2.b, p11.b/Z, p9.b[w14]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0176706 // ld1b { za0h.b[x15, #6] }, p1/Z, [x24, x23]\n" - ".inst 0x252e6d21 // psel p1.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6142 // psel p2.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0162306 // ld1b { za0h.b[x13, #6] }, p0/Z, [x24, x22]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0162ae7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x23, x22]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe01762a7 // ld1b { za0h.b[x15, #7] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" "add x10, x10, #0x10\n" - ".inst 0xe0bfcec0 // st1w { za0v.s[x14] }, p3/Z, [x22, XZR, LSL #2]\n" - "add x15, x15, #0x8\n" - ".inst 0xe0a8cac4 // st1w { za1v.s[x14] }, p2/Z, [x22, x8, LSL #2]\n" - ".inst 0xe0a9c6c1 // st1w { za0v.s[x14, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bcc2c5 // st1w { za1v.s[x14, #1] }, p0/Z, [x22, x28, LSL #2]\n" - "add x14, x14, #0x2\n" - "addvl x22, x22, #4\n" - "cmp x14, x17\n" + "add x13, x13, #0x8\n" + ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x21, x21, #4\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25376143 // psel p3.b, p8.b/Z, p10.b[w15, #2]\n" - ".inst 0x253f6142 // psel p2.b, p8.b/Z, p10.b[w15, #3]\n" - ".inst 0x25776141 // psel p1.b, p8.b/Z, p10.b[w15, #6]\n" - ".inst 0x257f6140 // psel p0.b, p8.b/Z, p10.b[w15, #7]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0176f62 // ld1b { za0h.b[x15, #2] }, p3/Z, [x27, x23]\n" - ".inst 0x25266d23 // psel p3.b, p11.b/Z, p9.b[w14]\n" - "ldr x27, [x11, #0x0]\n" - "mov x13, #0x0\n" - ".inst 0xe0176b23 // ld1b { za0h.b[x15, #3] }, p2/Z, [x25, x23]\n" - ".inst 0x25266d22 // psel p2.b, p11.b/Z, p9.b[w14]\n" - "ldr x25, [x10, #0x0]\n" - "mov x12, #0x0\n" - ".inst 0xe0176706 // ld1b { za0h.b[x15, #6] }, p1/Z, [x24, x23]\n" - ".inst 0x252e6d21 // psel p1.b, p11.b/Z, p9.b[w14, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162306 // ld1b { za0h.b[x13, #6] }, p0/Z, [x24, x22]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0x257d6141 // psel p1.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e7 // ld1b { za0h.b[x13, #7] }, p1/Z, [x23, x22]\n" "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08aa4 // st1w { za1v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + "whilelt p9.b, x14, %x[width]\n" + "incb x14\n" + ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" "add x11, x11, #0x10\n" - ".inst 0xe01762a7 // ld1b { za0h.b[x15, #7] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - "whilelt p9.b, x16, %x[width]\n" - ".inst 0xe0bfcec0 // st1w { za0v.s[x14] }, p3/Z, [x22, XZR, LSL #2]\n" - "incb x16\n" "add x10, x10, #0x10\n" - ".inst 0xe0a8cac4 // st1w { za1v.s[x14] }, p2/Z, [x22, x8, LSL #2]\n" - "incb x23\n" - "whilelt p8.b, x16, %x[width]\n" - ".inst 0xe0a9c6c1 // st1w { za0v.s[x14, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bcc2c5 // st1w { za1v.s[x14, #1] }, p0/Z, [x22, x28, LSL #2]\n" - "addvl x22, x22, #4\n" - "cbz x17, 7f\n" + ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "addvl x21, x21, #4\n" + "incb x22\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 7f\n" "6:" // K loop: Main loop: Second: Loop - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" - ".inst 0xe0172f60 // ld1b { za0h.b[x13] }, p3/Z, [x27, x23]\n" - ".inst 0x25246d23 // psel p3.b, p11.b/Z, p9.b[w12]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0172b21 // ld1b { za0h.b[x13, #1] }, p2/Z, [x25, x23]\n" - ".inst 0x25246d22 // psel p2.b, p11.b/Z, p9.b[w12]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0172704 // ld1b { za0h.b[x13, #4] }, p1/Z, [x24, x23]\n" - ".inst 0x252c6d21 // psel p1.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162120 // ld1b { za0h.b[x13] }, p0/Z, [x9, x22]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162341 // ld1b { za0h.b[x13, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6142 // psel p2.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0162304 // ld1b { za0h.b[x13, #4] }, p0/Z, [x24, x22]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0162ae5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x23, x22]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe01722a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" "add x10, x10, #0x10\n" - ".inst 0xe0bf8ec8 // st1w { za2v.s[x12] }, p3/Z, [x22, XZR, LSL #2]\n" "add x13, x13, #0x8\n" - ".inst 0xe0a88acc // st1w { za3v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bc82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" "add x12, x12, #0x2\n" - "addvl x22, x22, #4\n" - "cmp x12, x17\n" + "cmp x12, x15\n" + "addvl x21, x21, #4\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162120 // ld1b { za0h.b[x13] }, p0/Z, [x9, x22]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0xe0162341 // ld1b { za0h.b[x13, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0172f60 // ld1b { za0h.b[x13] }, p3/Z, [x27, x23]\n" - ".inst 0x25246d23 // psel p3.b, p11.b/Z, p9.b[w12]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0172b21 // ld1b { za0h.b[x13, #1] }, p2/Z, [x25, x23]\n" - ".inst 0x25246d22 // psel p2.b, p11.b/Z, p9.b[w12]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0172704 // ld1b { za0h.b[x13, #4] }, p1/Z, [x24, x23]\n" - ".inst 0x252c6d21 // psel p1.b, p11.b/Z, p9.b[w12, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162304 // ld1b { za0h.b[x13, #4] }, p0/Z, [x24, x22]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0x256d6141 // psel p1.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e5 // ld1b { za0h.b[x13, #5] }, p1/Z, [x23, x22]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe01722a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - "whilelt p9.b, x16, %x[width]\n" - ".inst 0xe0bf8ec8 // st1w { za2v.s[x12] }, p3/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08aac // st1w { za3v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + "whilelt p9.b, x14, %x[width]\n" "subs x20, x20, #0x1\n" + ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - ".inst 0xe0a88acc // st1w { za3v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" - "incb x16\n" - "incb x23\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bc82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x28, LSL #2]\n" - "addvl x22, x22, #4\n" + ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "addvl x21, x21, #4\n" + "incb x14\n" + "incb x22\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x26, 11f\n" + "cbnz x25, 11f\n" "mov x11, %x[in]\n" - "whilelt p8.b, x16, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "mov x13, #0x0\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - ".inst 0xe0bf8ec0 // st1w { za0v.s[x12] }, p3/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a88ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + "ldr x20, [x11, #0x0]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" "add x12, x12, #0x1\n" - "addvl x22, x22, #2\n" - "ldr x21, [x11, #0x0]\n" - "cmp x12, x8\n" - "ldr x20, [x11, x8, LSL #0x3]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0162283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x22]\n" + "cmp x12, x16\n" "add x11, x11, #0x8\n" - ".inst 0xe01726a2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x21, x23]\n" - ".inst 0xe0172283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x23]\n" + "addvl x21, x21, #2\n" "add x13, x13, #0x4\n" "blt 9b\n" - "whilelt p9.b, x16, %x[width]\n" - "whilelt p8.b, x16, %x[width]\n" + "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "mov x20, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - "add x20, x20, #0x4\n" - ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a882cc // st1w { za3v.s[x12] }, p0/Z, [x22, x8, LSL #2]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x22, x22, #2\n" - "cmp x12, x7\n" + "cmp x12, x17\n" + "addvl x21, x21, #2\n" + "add x20, x20, #0x4\n" "blt 10b\n" - "whilelt p8.b, x16, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" "12:" // K loop: Tails: Odd: Loop - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a882c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x8, LSL #2]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x22, x22, #2\n" - "cmp x12, x7\n" + "cmp x12, x17\n" + "addvl x21, x21, #2\n" "blt 12b\n" "13:" // K loop: End - "mov %x[out], x22\n" + "mov %x[out], x21\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp index 6930bf4056..27b9bc3806 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp @@ -32,321 +32,321 @@ void interleave_block<2, 4, VLType::SME, true>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" "mov x23, %x[width]\n" - "mov x21, %x[width]\n" - "cntb x20\n" - "incb x23\n" "mov z20.b, #0x1\n" + "incb x23\n" + "mov x20, %x[width]\n" "mov z19.s, #0x0\n" - "sub x7, x20, #0x1\n" - "cntw x8\n" "mov z18.s, #0x0\n" + "sub x17, x21, #0x1\n" + "cntw x16\n" "sub x23, x23, #0x1\n" - "ands x7, x21, x7\n" - "udiv x23, x23, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x7, x7, x20, NE\n" + "ands x17, x20, x17\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x21, NE\n" "lsl x22, %x[height], #0x1\n" // height * 2 - "lsl x21, x8, #0x1\n" + "lsl x21, x16, #0x1\n" "sub x20, x23, #0x1\n" - "add x7, x7, #0x3\n" + "add x17, x17, #0x3\n" "whilelt p9.b, XZR, x22\n" "whilelt p8.b, x21, x22\n" - "mov x17, #0x0\n" - "cntw x16, ALL, MUL #2\n" + "mov x15, #0x0\n" + "cntw x14, ALL, MUL #2\n" "cntw x11, ALL, MUL #3\n" "ptrue p4.b\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 "and x10, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) - "lsr x7, x7, #0x2\n" - "sub x9, x8, #0x2\n" + "lsr x17, x17, #0x2\n" + "sub x9, x16, #0x2\n" "ptrue p11.s\n" "zip1 p10.b, p9.b, p8.b\n" "mov x28, %x[row_offset]\n" "mov x27, %x[out]\n" - "whilelt p9.b, x17, %x[width]\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "cbnz %x[first], 1f\n" "addvl x27, x27, #-2\n" "ld1w { z19.s }, p4/Z, [x27]\n" "ld1w { z18.s }, p4/Z, [x27, #1, MUL VL]\n" "1:" // K loop: Load row sums: End "mov x26, %x[in]\n" - "add x25, %x[in], x8, LSL #3\n" + "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" - "mov x12, #0x0\n" "ldr x23, [x25, #0x0]\n" + "mov x12, #0x0\n" "ldr x22, [x26, #0x8]\n" "add x26, x26, #0x10\n" "ldr x21, [x25, #0x8]\n" "add x25, x25, #0x10\n" "cbz x9, 3f\n" "2:" // K loop: Charge: Loop - ".inst 0x25246143 // psel p3.b, p8.b/Z, p10.b[w12]\n" - ".inst 0x252c6142 // psel p2.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c02e1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x23, x28]\n" ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" - ".inst 0xe01c0f00 // ld1b { za0h.b[x12] }, p3/Z, [x24, x28]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe01c0ae1 // ld1b { za0h.b[x12, #1] }, p2/Z, [x23, x28]\n" "ldr x23, [x25, #0x0]\n" ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" "ldr x22, [x26, #0x8]\n" "add x26, x26, #0x10\n" ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" "add x12, x12, #0x8\n" + "cmp x12, x9, LSL #2\n" "ldr x21, [x25, #0x8]\n" "add x25, x25, #0x10\n" - "cmp x12, x9, LSL #2\n" "blt 2b\n" "3:" // K loop: Charge: End - ".inst 0x25246143 // psel p3.b, p8.b/Z, p10.b[w12]\n" - ".inst 0x252c6142 // psel p2.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe01c02e1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x23, x28]\n" ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" "mov x26, %x[in]\n" - "add x25, %x[in], x8, LSL #3\n" - ".inst 0xe01c0f00 // ld1b { za0h.b[x12] }, p3/Z, [x24, x28]\n" + "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" - "incb x17\n" - ".inst 0xe01c0ae1 // ld1b { za0h.b[x12, #1] }, p2/Z, [x23, x28]\n" + ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" "ldr x23, [x25, #0x0]\n" - ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" + "incb x28\n" + "incb x15\n" "ldr x22, [x26, #0x8]\n" "add x26, x26, #0x10\n" - ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" "add x25, x25, #0x10\n" - "incb x28\n" "cbz x20, 9f\n" "mov x20, x20\n" "4:" // K loop: Main loop - "whilelt p8.b, x17, %x[width]\n" - "mov x15, #0x0\n" - "mov x14, #0x0\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" "cbz x9, 6f\n" "5:" // K loop: Main loop: First: Loop - ".inst 0x25376143 // psel p3.b, p8.b/Z, p10.b[w15, #2]\n" - ".inst 0x253f6142 // psel p2.b, p8.b/Z, p10.b[w15, #3]\n" - ".inst 0x25776141 // psel p1.b, p8.b/Z, p10.b[w15, #6]\n" - ".inst 0x257f6140 // psel p0.b, p8.b/Z, p10.b[w15, #7]\n" - ".inst 0xe01c6f02 // ld1b { za0h.b[x15, #2] }, p3/Z, [x24, x28]\n" - ".inst 0x25266d23 // psel p3.b, p11.b/Z, p9.b[w14]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" "ldr x24, [x26, #0x0]\n" - ".inst 0xe01c6ae3 // ld1b { za0h.b[x15, #3] }, p2/Z, [x23, x28]\n" - ".inst 0x25266d22 // psel p2.b, p11.b/Z, p9.b[w14]\n" + ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6142 // psel p2.b, p8.b/Z, p10.b[w13, #7]\n" "ldr x23, [x25, #0x0]\n" - ".inst 0xe01c66c6 // ld1b { za0h.b[x15, #6] }, p1/Z, [x22, x28]\n" - ".inst 0x252e6d21 // psel p1.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0xe01c22c6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x22, x28]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - ".inst 0xe01c62a7 // ld1b { za0h.b[x15, #7] }, p0/Z, [x21, x28]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01c2aa7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - "add x25, x25, #0x10\n" - ".inst 0xe0bfcf60 // st1w { za0v.s[x14] }, p3/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc082d011 // mova z17.s, p4/M, za0v.s[x14]\n" - "add x15, x15, #0x8\n" - ".inst 0xc082d090 // mova z16.s, p4/M, za1v.s[x14]\n" + ".inst 0xe0bf8760 // st1w { za0v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" "sdot z19.s, z17.b, z20.b\n" - ".inst 0xe0a8cb64 // st1w { za1v.s[x14] }, p2/Z, [x27, x8, LSL #2]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "sdot z18.s, z16.b, z20.b\n" - ".inst 0xe0b0c761 // st1w { za0v.s[x14, #1] }, p1/Z, [x27, x16, LSL #2]\n" - ".inst 0xc082d031 // mova z17.s, p4/M, za0v.s[x14, #1]\n" - ".inst 0xe0abc365 // st1w { za1v.s[x14, #1] }, p0/Z, [x27, x11, LSL #2]\n" - ".inst 0xc082d0b0 // mova z16.s, p4/M, za1v.s[x14, #1]\n" - "add x14, x14, #0x2\n" - "addvl x27, x27, #4\n" - "cmp x14, x9\n" + ".inst 0xe0ae8361 // st1w { za0v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829031 // mova z17.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + ".inst 0xc08290b0 // mova z16.s, p4/M, za1v.s[x12, #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" "sdot z19.s, z17.b, z20.b\n" "sdot z18.s, z16.b, z20.b\n" + "addvl x27, x27, #4\n" + "add x13, x13, #0x8\n" "blt 5b\n" "6:" // K loop: Main loop: First: Tail - ".inst 0x25376143 // psel p3.b, p8.b/Z, p10.b[w15, #2]\n" - ".inst 0x253f6142 // psel p2.b, p8.b/Z, p10.b[w15, #3]\n" - ".inst 0x25776141 // psel p1.b, p8.b/Z, p10.b[w15, #6]\n" - ".inst 0x257f6140 // psel p0.b, p8.b/Z, p10.b[w15, #7]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6140 // psel p0.b, p8.b/Z, p10.b[w13, #7]\n" + ".inst 0xe01c26c6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x22, x28]\n" "mov x26, %x[in]\n" - "add x25, %x[in], x8, LSL #3\n" - ".inst 0xe01c6f02 // ld1b { za0h.b[x15, #2] }, p3/Z, [x24, x28]\n" - ".inst 0x25266d23 // psel p3.b, p11.b/Z, p9.b[w14]\n" + "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" - "mov x13, #0x0\n" - ".inst 0xe01c6ae3 // ld1b { za0h.b[x15, #3] }, p2/Z, [x23, x28]\n" - ".inst 0x25266d22 // psel p2.b, p11.b/Z, p9.b[w14]\n" + ".inst 0xe01c22a7 // ld1b { za0h.b[x13, #7] }, p0/Z, [x21, x28]\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "sdot z18.s, z16.b, z20.b\n" "ldr x23, [x25, #0x0]\n" - "mov x12, #0x0\n" - ".inst 0xe01c66c6 // ld1b { za0h.b[x15, #6] }, p1/Z, [x22, x28]\n" - ".inst 0x252e6d21 // psel p1.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - ".inst 0xe01c62a7 // ld1b { za0h.b[x15, #7] }, p0/Z, [x21, x28]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829031 // mova z17.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "ldr x21, [x25, #0x8]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - "whilelt p9.b, x17, %x[width]\n" - ".inst 0xc082d011 // mova z17.s, p4/M, za0v.s[x14]\n" - ".inst 0xe0bfcf60 // st1w { za0v.s[x14] }, p3/Z, [x27, XZR, LSL #2]\n" - "incb x17\n" - "add x25, x25, #0x10\n" - ".inst 0xc082d090 // mova z16.s, p4/M, za1v.s[x14]\n" + ".inst 0xe0bf8f60 // st1w { za0v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc08290b0 // mova z16.s, p4/M, za1v.s[x12, #1]\n" + "whilelt p9.b, x15, %x[width]\n" + ".inst 0xe0b08b64 // st1w { za1v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" + "incb x15\n" + "add x26, x26, #0x10\n" "sdot z19.s, z17.b, z20.b\n" - ".inst 0xe0a8cb64 // st1w { za1v.s[x14] }, p2/Z, [x27, x8, LSL #2]\n" - "incb x28\n" + ".inst 0xe0ae8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" + "add x25, x25, #0x10\n" "sdot z18.s, z16.b, z20.b\n" - ".inst 0xe0b0c761 // st1w { za0v.s[x14, #1] }, p1/Z, [x27, x16, LSL #2]\n" - ".inst 0xc082d031 // mova z17.s, p4/M, za0v.s[x14, #1]\n" - "whilelt p8.b, x17, %x[width]\n" - ".inst 0xc082d0b0 // mova z16.s, p4/M, za1v.s[x14, #1]\n" - ".inst 0xe0abc365 // st1w { za1v.s[x14, #1] }, p0/Z, [x27, x11, LSL #2]\n" + "incb x28\n" + ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" "addvl x27, x27, #4\n" - "sdot z19.s, z17.b, z20.b\n" - "sdot z18.s, z16.b, z20.b\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" "cbz x9, 8f\n" "7:" // K loop: Main loop: Second: Loop - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" - ".inst 0xe01c2f00 // ld1b { za0h.b[x13] }, p3/Z, [x24, x28]\n" - ".inst 0x25246d23 // psel p3.b, p11.b/Z, p9.b[w12]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c2300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x28]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" "ldr x24, [x26, #0x0]\n" - ".inst 0xe01c2ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x28]\n" - ".inst 0x25246d22 // psel p2.b, p11.b/Z, p9.b[w12]\n" + ".inst 0xe01c22e1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6142 // psel p2.b, p8.b/Z, p10.b[w13, #5]\n" "ldr x23, [x25, #0x0]\n" - ".inst 0xe01c26c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x28]\n" - ".inst 0x252c6d21 // psel p1.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0xe01c22c4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x22, x28]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01c2aa5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - "add x25, x25, #0x10\n" - ".inst 0xe0bf8f68 // st1w { za2v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xe0bf8768 // st1w { za2v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" - "add x13, x13, #0x8\n" ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" "sdot z19.s, z17.b, z20.b\n" - ".inst 0xe0a88b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x8, LSL #2]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "sdot z18.s, z16.b, z20.b\n" - ".inst 0xe0b08769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x16, LSL #2]\n" + ".inst 0xe0ae8369 // st1w { za2v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" ".inst 0xc0829131 // mova z17.s, p4/M, za2v.s[x12, #1]\n" ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" ".inst 0xc08291b0 // mova z16.s, p4/M, za3v.s[x12, #1]\n" "add x12, x12, #0x2\n" - "addvl x27, x27, #4\n" "cmp x12, x9\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" "sdot z19.s, z17.b, z20.b\n" "sdot z18.s, z16.b, z20.b\n" + "addvl x27, x27, #4\n" + "add x13, x13, #0x8\n" "blt 7b\n" "8:" // K loop: Main loop: Second: Tail - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c2300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x28]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0xe01c22e1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x23, x28]\n" ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" + ".inst 0xe01c26c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x28]\n" "mov x26, %x[in]\n" - "add x25, %x[in], x8, LSL #3\n" - ".inst 0xe01c2f00 // ld1b { za0h.b[x13] }, p3/Z, [x24, x28]\n" - ".inst 0x25246d23 // psel p3.b, p11.b/Z, p9.b[w12]\n" + "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" - ".inst 0xe01c2ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x28]\n" - ".inst 0x25246d22 // psel p2.b, p11.b/Z, p9.b[w12]\n" + ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" + ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" + ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" + "sdot z18.s, z16.b, z20.b\n" "ldr x23, [x25, #0x0]\n" - ".inst 0xe01c26c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x28]\n" - ".inst 0x252c6d21 // psel p1.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829131 // mova z17.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "ldr x21, [x25, #0x8]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - "whilelt p9.b, x17, %x[width]\n" - ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" ".inst 0xe0bf8f68 // st1w { za2v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc08291b0 // mova z16.s, p4/M, za3v.s[x12, #1]\n" + "whilelt p9.b, x15, %x[width]\n" + ".inst 0xe0b08b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" "subs x20, x20, #0x1\n" - "add x25, x25, #0x10\n" - ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" + "add x26, x26, #0x10\n" "sdot z19.s, z17.b, z20.b\n" - ".inst 0xe0a88b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x8, LSL #2]\n" - "incb x17\n" + ".inst 0xe0ae8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" + "add x25, x25, #0x10\n" "sdot z18.s, z16.b, z20.b\n" - ".inst 0xe0b08769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x16, LSL #2]\n" - ".inst 0xc0829131 // mova z17.s, p4/M, za2v.s[x12, #1]\n" - "incb x28\n" - ".inst 0xc08291b0 // mova z16.s, p4/M, za3v.s[x12, #1]\n" + "incb x15\n" ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" "addvl x27, x27, #4\n" - "sdot z19.s, z17.b, z20.b\n" - "sdot z18.s, z16.b, z20.b\n" + "incb x28\n" "bgt 4b\n" "9:" // K loop: Tails "cbnz x10, 12f\n" "mov x26, %x[in]\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "mov x13, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: First - ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "ldr x21, [x26, #0x0]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "ldr x20, [x26, x16, LSL #0x3]\n" + ".inst 0xe01c22a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x28]\n" + "add x12, x12, #0x1\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "cmp x12, x16\n" "sdot z19.s, z17.b, z20.b\n" "sdot z18.s, z16.b, z20.b\n" - ".inst 0xe0bf8f60 // st1w { za0v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" - ".inst 0xe0a88b64 // st1w { za1v.s[x12] }, p2/Z, [x27, x8, LSL #2]\n" - "add x12, x12, #0x1\n" - "addvl x27, x27, #2\n" - "ldr x21, [x26, #0x0]\n" - "cmp x12, x8\n" - "ldr x20, [x26, x8, LSL #0x3]\n" - "add x26, x26, #0x8\n" - ".inst 0xe01c26a2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x21, x28]\n" ".inst 0xe01c2283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x28]\n" + "add x26, x26, #0x8\n" + "addvl x27, x27, #2\n" "add x13, x13, #0x4\n" "blt 10b\n" - "whilelt p9.b, x17, %x[width]\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "mov x20, #0x0\n" "mov x12, #0x0\n" "11:" // K loop: Tails: Even: Second - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" "sdot z19.s, z17.b, z20.b\n" - "add x20, x20, #0x4\n" "sdot z18.s, z16.b, z20.b\n" - ".inst 0xe0bf8768 // st1w { za2v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" - ".inst 0xe0a8836c // st1w { za3v.s[x12] }, p0/Z, [x27, x8, LSL #2]\n" - "add x12, x12, #0x1\n" "addvl x27, x27, #2\n" - "cmp x12, x7\n" + "add x20, x20, #0x4\n" "blt 11b\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "b 14f\n" "12:" // K loop: Tails: Odd "mov x12, #0x0\n" "13:" // K loop: Tails: Odd: Loop - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" "sdot z19.s, z17.b, z20.b\n" "sdot z18.s, z16.b, z20.b\n" - ".inst 0xe0bf8760 // st1w { za0v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" - ".inst 0xe0a88364 // st1w { za1v.s[x12] }, p0/Z, [x27, x8, LSL #2]\n" - "add x12, x12, #0x1\n" "addvl x27, x27, #2\n" - "cmp x12, x7\n" "blt 13b\n" "14:" // K loop: End "st1w { z19.s }, p4, [x27]\n" "st1w { z18.s }, p4, [x27, #1, MUL VL]\n" "addvl x27, x27, #2\n" - ".inst 0xd503467f // SMSTOP\n" "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp index 9ce93ed95c..3f3863720a 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp @@ -32,265 +32,265 @@ void interleave_block<2, 4, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" "mov x23, %x[width]\n" - "mov x21, %x[width]\n" - "cntb x20\n" "incb x23\n" - "sub x7, x20, #0x1\n" - "cntw x8\n" + "mov x20, %x[width]\n" + "sub x17, x21, #0x1\n" + "cntw x16\n" "sub x23, x23, #0x1\n" - "ands x7, x21, x7\n" - "udiv x23, x23, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x7, x7, x20, NE\n" + "ands x17, x20, x17\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x21, NE\n" "lsl x22, %x[height], #0x1\n" // height * 2 - "lsl x21, x8, #0x1\n" + "lsl x21, x16, #0x1\n" "sub x20, x23, #0x1\n" - "add x7, x7, #0x3\n" - "sub x17, x8, #0x2\n" + "add x17, x17, #0x3\n" + "sub x15, x16, #0x2\n" "whilelt p9.b, XZR, x22\n" "whilelt p8.b, x21, x22\n" - "mov x16, #0x0\n" + "mov x14, #0x0\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - "cntw x9, ALL, MUL #2\n" - "cntw x28, ALL, MUL #3\n" - "ldr x27, [x11, #0x0]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + "cntw x28, ALL, MUL #2\n" + "cntw x27, ALL, MUL #3\n" + "ldr x26, [x10, #0x0]\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x26, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) - "ldr x25, [x10, #0x0]\n" - "lsr x7, x7, #0x2\n" - "ptrue p11.s\n" + "and x25, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) "ldr x24, [x11, #0x8]\n" + "lsr x17, x17, #0x2\n" + "ptrue p11.s\n" + "ldr x23, [x10, #0x8]\n" "zip1 p10.b, p9.b, p8.b\n" - "mov x23, %x[row_offset]\n" - "ldr x21, [x10, #0x8]\n" - "mov x22, %x[out]\n" - "whilelt p9.b, x16, %x[width]\n" - "whilelt p8.b, x16, %x[width]\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" "mov x12, #0x0\n" - "cbz x17, 2f\n" + "cbz x15, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25246143 // psel p3.b, p8.b/Z, p10.b[w12]\n" - ".inst 0x252c6142 // psel p2.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160120 // ld1b { za0h.b[x12] }, p0/Z, [x9, x22]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0160341 // ld1b { za0h.b[x12, #1] }, p0/Z, [x26, x22]\n" ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" - ".inst 0xe0170f60 // ld1b { za0h.b[x12] }, p3/Z, [x27, x23]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0170b21 // ld1b { za0h.b[x12, #1] }, p2/Z, [x25, x23]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0170704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x23]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0160704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x22]\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - ".inst 0xe01702a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x23]\n" + ".inst 0xe01602e5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x23, x22]\n" "add x12, x12, #0x8\n" - "ldr x21, [x10, #0x8]\n" + "cmp x12, x15, LSL #2\n" + "ldr x23, [x10, #0x8]\n" "add x10, x10, #0x10\n" - "cmp x12, x17, LSL #2\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25246143 // psel p3.b, p8.b/Z, p10.b[w12]\n" - ".inst 0x252c6142 // psel p2.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160120 // ld1b { za0h.b[x12] }, p0/Z, [x9, x22]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe0160341 // ld1b { za0h.b[x12, #1] }, p0/Z, [x26, x22]\n" ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + ".inst 0xe0160704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x22]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0170f60 // ld1b { za0h.b[x12] }, p3/Z, [x27, x23]\n" - "ldr x27, [x11, #0x0]\n" - "incb x16\n" - ".inst 0xe0170b21 // ld1b { za0h.b[x12, #1] }, p2/Z, [x25, x23]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0170704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x23]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe01602e5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x23, x22]\n" + "ldr x26, [x10, #0x0]\n" + "incb x22\n" + "incb x14\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - ".inst 0xe01702a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" + "ldr x23, [x10, #0x8]\n" "add x10, x10, #0x10\n" - "incb x23\n" "cbz x20, 8f\n" "mov x20, x20\n" "3:" // K loop: Main loop - "whilelt p8.b, x16, %x[width]\n" - "mov x15, #0x0\n" - "mov x14, #0x0\n" - "cbz x17, 5f\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25376143 // psel p3.b, p8.b/Z, p10.b[w15, #2]\n" - ".inst 0x253f6142 // psel p2.b, p8.b/Z, p10.b[w15, #3]\n" - ".inst 0x25776141 // psel p1.b, p8.b/Z, p10.b[w15, #6]\n" - ".inst 0x257f6140 // psel p0.b, p8.b/Z, p10.b[w15, #7]\n" - ".inst 0xe0176f62 // ld1b { za0h.b[x15, #2] }, p3/Z, [x27, x23]\n" - ".inst 0x25266d23 // psel p3.b, p11.b/Z, p9.b[w14]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0176b23 // ld1b { za0h.b[x15, #3] }, p2/Z, [x25, x23]\n" - ".inst 0x25266d22 // psel p2.b, p11.b/Z, p9.b[w14]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0176706 // ld1b { za0h.b[x15, #6] }, p1/Z, [x24, x23]\n" - ".inst 0x252e6d21 // psel p1.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6142 // psel p2.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0162306 // ld1b { za0h.b[x13, #6] }, p0/Z, [x24, x22]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0162ae7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x23, x22]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe01762a7 // ld1b { za0h.b[x15, #7] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" "add x10, x10, #0x10\n" - ".inst 0xe0bfcec0 // st1w { za0v.s[x14] }, p3/Z, [x22, XZR, LSL #2]\n" - "add x15, x15, #0x8\n" - ".inst 0xe0a8cac4 // st1w { za1v.s[x14] }, p2/Z, [x22, x8, LSL #2]\n" - ".inst 0xe0a9c6c1 // st1w { za0v.s[x14, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bcc2c5 // st1w { za1v.s[x14, #1] }, p0/Z, [x22, x28, LSL #2]\n" - "add x14, x14, #0x2\n" - "addvl x22, x22, #4\n" - "cmp x14, x17\n" + "add x13, x13, #0x8\n" + ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x21, x21, #4\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25376143 // psel p3.b, p8.b/Z, p10.b[w15, #2]\n" - ".inst 0x253f6142 // psel p2.b, p8.b/Z, p10.b[w15, #3]\n" - ".inst 0x25776141 // psel p1.b, p8.b/Z, p10.b[w15, #6]\n" - ".inst 0x257f6140 // psel p0.b, p8.b/Z, p10.b[w15, #7]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0176f62 // ld1b { za0h.b[x15, #2] }, p3/Z, [x27, x23]\n" - ".inst 0x25266d23 // psel p3.b, p11.b/Z, p9.b[w14]\n" - "ldr x27, [x11, #0x0]\n" - "mov x13, #0x0\n" - ".inst 0xe0176b23 // ld1b { za0h.b[x15, #3] }, p2/Z, [x25, x23]\n" - ".inst 0x25266d22 // psel p2.b, p11.b/Z, p9.b[w14]\n" - "ldr x25, [x10, #0x0]\n" - "mov x12, #0x0\n" - ".inst 0xe0176706 // ld1b { za0h.b[x15, #6] }, p1/Z, [x24, x23]\n" - ".inst 0x252e6d21 // psel p1.b, p11.b/Z, p9.b[w14, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162306 // ld1b { za0h.b[x13, #6] }, p0/Z, [x24, x22]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0x257d6141 // psel p1.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e7 // ld1b { za0h.b[x13, #7] }, p1/Z, [x23, x22]\n" "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08aa4 // st1w { za1v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + "whilelt p9.b, x14, %x[width]\n" + "incb x14\n" + ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" "add x11, x11, #0x10\n" - ".inst 0xe01762a7 // ld1b { za0h.b[x15, #7] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - "whilelt p9.b, x16, %x[width]\n" - ".inst 0xe0bfcec0 // st1w { za0v.s[x14] }, p3/Z, [x22, XZR, LSL #2]\n" - "incb x16\n" "add x10, x10, #0x10\n" - ".inst 0xe0a8cac4 // st1w { za1v.s[x14] }, p2/Z, [x22, x8, LSL #2]\n" - "incb x23\n" - "whilelt p8.b, x16, %x[width]\n" - ".inst 0xe0a9c6c1 // st1w { za0v.s[x14, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bcc2c5 // st1w { za1v.s[x14, #1] }, p0/Z, [x22, x28, LSL #2]\n" - "addvl x22, x22, #4\n" - "cbz x17, 7f\n" + ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "addvl x21, x21, #4\n" + "incb x22\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 7f\n" "6:" // K loop: Main loop: Second: Loop - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" - ".inst 0xe0172f60 // ld1b { za0h.b[x13] }, p3/Z, [x27, x23]\n" - ".inst 0x25246d23 // psel p3.b, p11.b/Z, p9.b[w12]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0172b21 // ld1b { za0h.b[x13, #1] }, p2/Z, [x25, x23]\n" - ".inst 0x25246d22 // psel p2.b, p11.b/Z, p9.b[w12]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0172704 // ld1b { za0h.b[x13, #4] }, p1/Z, [x24, x23]\n" - ".inst 0x252c6d21 // psel p1.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162120 // ld1b { za0h.b[x13] }, p0/Z, [x9, x22]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162341 // ld1b { za0h.b[x13, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6142 // psel p2.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0162304 // ld1b { za0h.b[x13, #4] }, p0/Z, [x24, x22]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0162ae5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x23, x22]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe01722a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" "add x10, x10, #0x10\n" - ".inst 0xe0bf8ec8 // st1w { za2v.s[x12] }, p3/Z, [x22, XZR, LSL #2]\n" "add x13, x13, #0x8\n" - ".inst 0xe0a88acc // st1w { za3v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bc82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" "add x12, x12, #0x2\n" - "addvl x22, x22, #4\n" - "cmp x12, x17\n" + "cmp x12, x15\n" + "addvl x21, x21, #4\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162120 // ld1b { za0h.b[x13] }, p0/Z, [x9, x22]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0xe0162341 // ld1b { za0h.b[x13, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x8, LSL #3\n" - ".inst 0xe0172f60 // ld1b { za0h.b[x13] }, p3/Z, [x27, x23]\n" - ".inst 0x25246d23 // psel p3.b, p11.b/Z, p9.b[w12]\n" - "ldr x27, [x11, #0x0]\n" - ".inst 0xe0172b21 // ld1b { za0h.b[x13, #1] }, p2/Z, [x25, x23]\n" - ".inst 0x25246d22 // psel p2.b, p11.b/Z, p9.b[w12]\n" - "ldr x25, [x10, #0x0]\n" - ".inst 0xe0172704 // ld1b { za0h.b[x13, #4] }, p1/Z, [x24, x23]\n" - ".inst 0x252c6d21 // psel p1.b, p11.b/Z, p9.b[w12, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162304 // ld1b { za0h.b[x13, #4] }, p0/Z, [x24, x22]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0x256d6141 // psel p1.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e5 // ld1b { za0h.b[x13, #5] }, p1/Z, [x23, x22]\n" "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe01722a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x23]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - "whilelt p9.b, x16, %x[width]\n" - ".inst 0xe0bf8ec8 // st1w { za2v.s[x12] }, p3/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08aac // st1w { za3v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + "whilelt p9.b, x14, %x[width]\n" "subs x20, x20, #0x1\n" + ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - ".inst 0xe0a88acc // st1w { za3v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" - "incb x16\n" - "incb x23\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bc82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x28, LSL #2]\n" - "addvl x22, x22, #4\n" + ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "addvl x21, x21, #4\n" + "incb x14\n" + "incb x22\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x26, 11f\n" + "cbnz x25, 11f\n" "mov x11, %x[in]\n" - "whilelt p8.b, x16, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "mov x13, #0x0\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - ".inst 0xe0bf8ec0 // st1w { za0v.s[x12] }, p3/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a88ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x8, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + "ldr x20, [x11, #0x0]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" "add x12, x12, #0x1\n" - "addvl x22, x22, #2\n" - "ldr x21, [x11, #0x0]\n" - "cmp x12, x8\n" - "ldr x20, [x11, x8, LSL #0x3]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0162283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x22]\n" + "cmp x12, x16\n" "add x11, x11, #0x8\n" - ".inst 0xe01726a2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x21, x23]\n" - ".inst 0xe0172283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x23]\n" + "addvl x21, x21, #2\n" "add x13, x13, #0x4\n" "blt 9b\n" - "whilelt p9.b, x16, %x[width]\n" - "whilelt p8.b, x16, %x[width]\n" + "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "mov x20, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - "add x20, x20, #0x4\n" - ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a882cc // st1w { za3v.s[x12] }, p0/Z, [x22, x8, LSL #2]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x22, x22, #2\n" - "cmp x12, x7\n" + "cmp x12, x17\n" + "addvl x21, x21, #2\n" + "add x20, x20, #0x4\n" "blt 10b\n" - "whilelt p8.b, x16, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" "12:" // K loop: Tails: Odd: Loop - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0a882c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x8, LSL #2]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x22, x22, #2\n" - "cmp x12, x7\n" + "cmp x12, x17\n" + "addvl x21, x21, #2\n" "blt 12b\n" "13:" // K loop: End - "mov %x[out], x22\n" + "mov %x[out], x21\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp index 915381334e..c8657fad1c 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp @@ -32,321 +32,321 @@ void interleave_block<2, 4, VLType::SME, true>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" "mov x23, %x[width]\n" - "mov x21, %x[width]\n" - "cntb x20\n" - "incb x23\n" "mov z20.b, #0x1\n" + "incb x23\n" + "mov x20, %x[width]\n" "mov z19.s, #0x0\n" - "sub x7, x20, #0x1\n" - "cntw x8\n" "mov z18.s, #0x0\n" + "sub x17, x21, #0x1\n" + "cntw x16\n" "sub x23, x23, #0x1\n" - "ands x7, x21, x7\n" - "udiv x23, x23, x20\n" // n_passes = ceildiv(width, VL<T>) - "csel x7, x7, x20, NE\n" + "ands x17, x20, x17\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x21, NE\n" "lsl x22, %x[height], #0x1\n" // height * 2 - "lsl x21, x8, #0x1\n" + "lsl x21, x16, #0x1\n" "sub x20, x23, #0x1\n" - "add x7, x7, #0x3\n" + "add x17, x17, #0x3\n" "whilelt p9.b, XZR, x22\n" "whilelt p8.b, x21, x22\n" - "mov x17, #0x0\n" - "cntw x16, ALL, MUL #2\n" + "mov x15, #0x0\n" + "cntw x14, ALL, MUL #2\n" "cntw x11, ALL, MUL #3\n" "ptrue p4.b\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 "and x10, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) - "lsr x7, x7, #0x2\n" - "sub x9, x8, #0x2\n" + "lsr x17, x17, #0x2\n" + "sub x9, x16, #0x2\n" "ptrue p11.s\n" "zip1 p10.b, p9.b, p8.b\n" "mov x28, %x[row_offset]\n" "mov x27, %x[out]\n" - "whilelt p9.b, x17, %x[width]\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "cbnz %x[first], 1f\n" "addvl x27, x27, #-2\n" "ld1w { z19.s }, p4/Z, [x27]\n" "ld1w { z18.s }, p4/Z, [x27, #1, MUL VL]\n" "1:" // K loop: Load row sums: End "mov x26, %x[in]\n" - "add x25, %x[in], x8, LSL #3\n" + "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" - "mov x12, #0x0\n" "ldr x23, [x25, #0x0]\n" + "mov x12, #0x0\n" "ldr x22, [x26, #0x8]\n" "add x26, x26, #0x10\n" "ldr x21, [x25, #0x8]\n" "add x25, x25, #0x10\n" "cbz x9, 3f\n" "2:" // K loop: Charge: Loop - ".inst 0x25246143 // psel p3.b, p8.b/Z, p10.b[w12]\n" - ".inst 0x252c6142 // psel p2.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c02e1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x23, x28]\n" ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" - ".inst 0xe01c0f00 // ld1b { za0h.b[x12] }, p3/Z, [x24, x28]\n" - "ldr x24, [x26, #0x0]\n" - ".inst 0xe01c0ae1 // ld1b { za0h.b[x12, #1] }, p2/Z, [x23, x28]\n" "ldr x23, [x25, #0x0]\n" ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" "ldr x22, [x26, #0x8]\n" "add x26, x26, #0x10\n" ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" "add x12, x12, #0x8\n" + "cmp x12, x9, LSL #2\n" "ldr x21, [x25, #0x8]\n" "add x25, x25, #0x10\n" - "cmp x12, x9, LSL #2\n" "blt 2b\n" "3:" // K loop: Charge: End - ".inst 0x25246143 // psel p3.b, p8.b/Z, p10.b[w12]\n" - ".inst 0x252c6142 // psel p2.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe01c02e1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x23, x28]\n" ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" "mov x26, %x[in]\n" - "add x25, %x[in], x8, LSL #3\n" - ".inst 0xe01c0f00 // ld1b { za0h.b[x12] }, p3/Z, [x24, x28]\n" + "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" - "incb x17\n" - ".inst 0xe01c0ae1 // ld1b { za0h.b[x12, #1] }, p2/Z, [x23, x28]\n" + ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" "ldr x23, [x25, #0x0]\n" - ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" + "incb x28\n" + "incb x15\n" "ldr x22, [x26, #0x8]\n" "add x26, x26, #0x10\n" - ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" "add x25, x25, #0x10\n" - "incb x28\n" "cbz x20, 9f\n" "mov x20, x20\n" "4:" // K loop: Main loop - "whilelt p8.b, x17, %x[width]\n" - "mov x15, #0x0\n" - "mov x14, #0x0\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" "cbz x9, 6f\n" "5:" // K loop: Main loop: First: Loop - ".inst 0x25376143 // psel p3.b, p8.b/Z, p10.b[w15, #2]\n" - ".inst 0x253f6142 // psel p2.b, p8.b/Z, p10.b[w15, #3]\n" - ".inst 0x25776141 // psel p1.b, p8.b/Z, p10.b[w15, #6]\n" - ".inst 0x257f6140 // psel p0.b, p8.b/Z, p10.b[w15, #7]\n" - ".inst 0xe01c6f02 // ld1b { za0h.b[x15, #2] }, p3/Z, [x24, x28]\n" - ".inst 0x25266d23 // psel p3.b, p11.b/Z, p9.b[w14]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" "ldr x24, [x26, #0x0]\n" - ".inst 0xe01c6ae3 // ld1b { za0h.b[x15, #3] }, p2/Z, [x23, x28]\n" - ".inst 0x25266d22 // psel p2.b, p11.b/Z, p9.b[w14]\n" + ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6142 // psel p2.b, p8.b/Z, p10.b[w13, #7]\n" "ldr x23, [x25, #0x0]\n" - ".inst 0xe01c66c6 // ld1b { za0h.b[x15, #6] }, p1/Z, [x22, x28]\n" - ".inst 0x252e6d21 // psel p1.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0xe01c22c6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x22, x28]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - ".inst 0xe01c62a7 // ld1b { za0h.b[x15, #7] }, p0/Z, [x21, x28]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01c2aa7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - "add x25, x25, #0x10\n" - ".inst 0xe0bfcf60 // st1w { za0v.s[x14] }, p3/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc082d010 // mova z16.s, p4/M, za0v.s[x14]\n" - "add x15, x15, #0x8\n" - ".inst 0xc082d091 // mova z17.s, p4/M, za1v.s[x14]\n" + ".inst 0xe0bf8760 // st1w { za0v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" "udot z19.s, z16.b, z20.b\n" - ".inst 0xe0a8cb64 // st1w { za1v.s[x14] }, p2/Z, [x27, x8, LSL #2]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "udot z18.s, z17.b, z20.b\n" - ".inst 0xe0b0c761 // st1w { za0v.s[x14, #1] }, p1/Z, [x27, x16, LSL #2]\n" - ".inst 0xc082d030 // mova z16.s, p4/M, za0v.s[x14, #1]\n" - ".inst 0xe0abc365 // st1w { za1v.s[x14, #1] }, p0/Z, [x27, x11, LSL #2]\n" - ".inst 0xc082d0b1 // mova z17.s, p4/M, za1v.s[x14, #1]\n" - "add x14, x14, #0x2\n" - "addvl x27, x27, #4\n" - "cmp x14, x9\n" + ".inst 0xe0ae8361 // st1w { za0v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829030 // mova z16.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + ".inst 0xc08290b1 // mova z17.s, p4/M, za1v.s[x12, #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" "udot z19.s, z16.b, z20.b\n" "udot z18.s, z17.b, z20.b\n" + "addvl x27, x27, #4\n" + "add x13, x13, #0x8\n" "blt 5b\n" "6:" // K loop: Main loop: First: Tail - ".inst 0x25376143 // psel p3.b, p8.b/Z, p10.b[w15, #2]\n" - ".inst 0x253f6142 // psel p2.b, p8.b/Z, p10.b[w15, #3]\n" - ".inst 0x25776141 // psel p1.b, p8.b/Z, p10.b[w15, #6]\n" - ".inst 0x257f6140 // psel p0.b, p8.b/Z, p10.b[w15, #7]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6140 // psel p0.b, p8.b/Z, p10.b[w13, #7]\n" + ".inst 0xe01c26c6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x22, x28]\n" "mov x26, %x[in]\n" - "add x25, %x[in], x8, LSL #3\n" - ".inst 0xe01c6f02 // ld1b { za0h.b[x15, #2] }, p3/Z, [x24, x28]\n" - ".inst 0x25266d23 // psel p3.b, p11.b/Z, p9.b[w14]\n" + "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" - "mov x13, #0x0\n" - ".inst 0xe01c6ae3 // ld1b { za0h.b[x15, #3] }, p2/Z, [x23, x28]\n" - ".inst 0x25266d22 // psel p2.b, p11.b/Z, p9.b[w14]\n" + ".inst 0xe01c22a7 // ld1b { za0h.b[x13, #7] }, p0/Z, [x21, x28]\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "udot z18.s, z17.b, z20.b\n" "ldr x23, [x25, #0x0]\n" - "mov x12, #0x0\n" - ".inst 0xe01c66c6 // ld1b { za0h.b[x15, #6] }, p1/Z, [x22, x28]\n" - ".inst 0x252e6d21 // psel p1.b, p11.b/Z, p9.b[w14, #1]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - ".inst 0xe01c62a7 // ld1b { za0h.b[x15, #7] }, p0/Z, [x21, x28]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829030 // mova z16.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "ldr x21, [x25, #0x8]\n" - ".inst 0x252e6d20 // psel p0.b, p11.b/Z, p9.b[w14, #1]\n" - "whilelt p9.b, x17, %x[width]\n" - ".inst 0xc082d010 // mova z16.s, p4/M, za0v.s[x14]\n" - ".inst 0xe0bfcf60 // st1w { za0v.s[x14] }, p3/Z, [x27, XZR, LSL #2]\n" - "incb x17\n" - "add x25, x25, #0x10\n" - ".inst 0xc082d091 // mova z17.s, p4/M, za1v.s[x14]\n" + ".inst 0xe0bf8f60 // st1w { za0v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc08290b1 // mova z17.s, p4/M, za1v.s[x12, #1]\n" + "whilelt p9.b, x15, %x[width]\n" + ".inst 0xe0b08b64 // st1w { za1v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" + "incb x15\n" + "add x26, x26, #0x10\n" "udot z19.s, z16.b, z20.b\n" - ".inst 0xe0a8cb64 // st1w { za1v.s[x14] }, p2/Z, [x27, x8, LSL #2]\n" - "incb x28\n" + ".inst 0xe0ae8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" + "add x25, x25, #0x10\n" "udot z18.s, z17.b, z20.b\n" - ".inst 0xe0b0c761 // st1w { za0v.s[x14, #1] }, p1/Z, [x27, x16, LSL #2]\n" - ".inst 0xc082d030 // mova z16.s, p4/M, za0v.s[x14, #1]\n" - "whilelt p8.b, x17, %x[width]\n" - ".inst 0xc082d0b1 // mova z17.s, p4/M, za1v.s[x14, #1]\n" - ".inst 0xe0abc365 // st1w { za1v.s[x14, #1] }, p0/Z, [x27, x11, LSL #2]\n" + "incb x28\n" + ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" "addvl x27, x27, #4\n" - "udot z19.s, z16.b, z20.b\n" - "udot z18.s, z17.b, z20.b\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" "cbz x9, 8f\n" "7:" // K loop: Main loop: Second: Loop - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" - ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" - ".inst 0xe01c2f00 // ld1b { za0h.b[x13] }, p3/Z, [x24, x28]\n" - ".inst 0x25246d23 // psel p3.b, p11.b/Z, p9.b[w12]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c2300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x28]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" "ldr x24, [x26, #0x0]\n" - ".inst 0xe01c2ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x28]\n" - ".inst 0x25246d22 // psel p2.b, p11.b/Z, p9.b[w12]\n" + ".inst 0xe01c22e1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6142 // psel p2.b, p8.b/Z, p10.b[w13, #5]\n" "ldr x23, [x25, #0x0]\n" - ".inst 0xe01c26c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x28]\n" - ".inst 0x252c6d21 // psel p1.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0xe01c22c4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x22, x28]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01c2aa5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - "add x25, x25, #0x10\n" - ".inst 0xe0bf8f68 // st1w { za2v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xe0bf8768 // st1w { za2v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" - "add x13, x13, #0x8\n" ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" "udot z19.s, z16.b, z20.b\n" - ".inst 0xe0a88b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x8, LSL #2]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "udot z18.s, z17.b, z20.b\n" - ".inst 0xe0b08769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x16, LSL #2]\n" + ".inst 0xe0ae8369 // st1w { za2v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" ".inst 0xc08291b1 // mova z17.s, p4/M, za3v.s[x12, #1]\n" "add x12, x12, #0x2\n" - "addvl x27, x27, #4\n" "cmp x12, x9\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" "udot z19.s, z16.b, z20.b\n" "udot z18.s, z17.b, z20.b\n" + "addvl x27, x27, #4\n" + "add x13, x13, #0x8\n" "blt 7b\n" "8:" // K loop: Main loop: Second: Tail - ".inst 0x25256143 // psel p3.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c2300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x28]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0xe01c22e1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x23, x28]\n" ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" + ".inst 0xe01c26c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x28]\n" "mov x26, %x[in]\n" - "add x25, %x[in], x8, LSL #3\n" - ".inst 0xe01c2f00 // ld1b { za0h.b[x13] }, p3/Z, [x24, x28]\n" - ".inst 0x25246d23 // psel p3.b, p11.b/Z, p9.b[w12]\n" + "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" - ".inst 0xe01c2ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x28]\n" - ".inst 0x25246d22 // psel p2.b, p11.b/Z, p9.b[w12]\n" + ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" + ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" + ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" + "udot z18.s, z17.b, z20.b\n" "ldr x23, [x25, #0x0]\n" - ".inst 0xe01c26c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x28]\n" - ".inst 0x252c6d21 // psel p1.b, p11.b/Z, p9.b[w12, #1]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" - "add x26, x26, #0x10\n" - ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "ldr x21, [x25, #0x8]\n" - ".inst 0x252c6d20 // psel p0.b, p11.b/Z, p9.b[w12, #1]\n" - "whilelt p9.b, x17, %x[width]\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" ".inst 0xe0bf8f68 // st1w { za2v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc08291b1 // mova z17.s, p4/M, za3v.s[x12, #1]\n" + "whilelt p9.b, x15, %x[width]\n" + ".inst 0xe0b08b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" "subs x20, x20, #0x1\n" - "add x25, x25, #0x10\n" - ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" + "add x26, x26, #0x10\n" "udot z19.s, z16.b, z20.b\n" - ".inst 0xe0a88b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x8, LSL #2]\n" - "incb x17\n" + ".inst 0xe0ae8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" + "add x25, x25, #0x10\n" "udot z18.s, z17.b, z20.b\n" - ".inst 0xe0b08769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x16, LSL #2]\n" - ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" - "incb x28\n" - ".inst 0xc08291b1 // mova z17.s, p4/M, za3v.s[x12, #1]\n" + "incb x15\n" ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" "addvl x27, x27, #4\n" - "udot z19.s, z16.b, z20.b\n" - "udot z18.s, z17.b, z20.b\n" + "incb x28\n" "bgt 4b\n" "9:" // K loop: Tails "cbnz x10, 12f\n" "mov x26, %x[in]\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "mov x13, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: First - ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "ldr x21, [x26, #0x0]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "ldr x20, [x26, x16, LSL #0x3]\n" + ".inst 0xe01c22a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x28]\n" + "add x12, x12, #0x1\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "cmp x12, x16\n" "udot z19.s, z16.b, z20.b\n" "udot z18.s, z17.b, z20.b\n" - ".inst 0xe0bf8f60 // st1w { za0v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" - ".inst 0xe0a88b64 // st1w { za1v.s[x12] }, p2/Z, [x27, x8, LSL #2]\n" - "add x12, x12, #0x1\n" - "addvl x27, x27, #2\n" - "ldr x21, [x26, #0x0]\n" - "cmp x12, x8\n" - "ldr x20, [x26, x8, LSL #0x3]\n" - "add x26, x26, #0x8\n" - ".inst 0xe01c26a2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x21, x28]\n" ".inst 0xe01c2283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x28]\n" + "add x26, x26, #0x8\n" + "addvl x27, x27, #2\n" "add x13, x13, #0x4\n" "blt 10b\n" - "whilelt p9.b, x17, %x[width]\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "mov x20, #0x0\n" "mov x12, #0x0\n" "11:" // K loop: Tails: Even: Second - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" "udot z19.s, z16.b, z20.b\n" - "add x20, x20, #0x4\n" "udot z18.s, z17.b, z20.b\n" - ".inst 0xe0bf8768 // st1w { za2v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" - ".inst 0xe0a8836c // st1w { za3v.s[x12] }, p0/Z, [x27, x8, LSL #2]\n" - "add x12, x12, #0x1\n" "addvl x27, x27, #2\n" - "cmp x12, x7\n" + "add x20, x20, #0x4\n" "blt 11b\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "b 14f\n" "12:" // K loop: Tails: Odd "mov x12, #0x0\n" "13:" // K loop: Tails: Odd: Loop - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" "udot z19.s, z16.b, z20.b\n" "udot z18.s, z17.b, z20.b\n" - ".inst 0xe0bf8760 // st1w { za0v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" - ".inst 0xe0a88364 // st1w { za1v.s[x12] }, p0/Z, [x27, x8, LSL #2]\n" - "add x12, x12, #0x1\n" "addvl x27, x27, #2\n" - "cmp x12, x7\n" "blt 13b\n" "14:" // K loop: End "st1w { z19.s }, p4, [x27]\n" "st1w { z18.s }, p4, [x27, #1, MUL VL]\n" "addvl x27, x27, #2\n" - ".inst 0xd503467f // SMSTOP\n" "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp index 19d87039fe..9403efc7c6 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp @@ -32,66 +32,66 @@ void interleave_block<2, 1, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x28, #0x0\n" - "mov x27, %x[row_offset]\n" - "cnth x26\n" - "cnth x25\n" - "cmp %x[height], x26\n" + "cnth x28\n" + "cmp %x[height], x28\n" + "cnth x27\n" + "csel x28, %x[height], x28, LT\n" + "mov x26, #0x0\n" "ptrue p13.s\n" - "csel x26, %x[height], x26, LT\n" + "sub x28, x28, #0x1\n" "whilelt p12.h, XZR, %x[height]\n" - "sub x26, x26, #0x1\n" - "whilelt p11.h, x25, %x[height]\n" + "whilelt p11.h, x27, %x[height]\n" + "mov x25, %x[row_offset]\n" "mov x24, %x[out]\n" - "whilelt p10.h, x28, %x[width]\n" - "whilelt p9.h, x28, %x[width]\n" - "whilelt p8.h, x28, %x[width]\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" "1:" // Width loop "add x23, %x[in], XZR, LSL #3\n" - "add x20, %x[in], x25, LSL #3\n" - "mov x13, #0x0\n" + "add x20, %x[in], x27, LSL #3\n" "ldr x22, [x23], #0x8\n" + "mov x12, #0x0\n" "ldr x21, [x20], #0x8\n" - "cbz x26, 3f\n" + "cbz x28, 3f\n" "2:" // Loads: Loop - ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0xe05b26c0 // ld1h { za0h.h[x13] }, p1/Z, [x22, x27, LSL #1]\n" + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05906c0 // ld1h { za0h.h[x12] }, p1/Z, [x22, x25, LSL #1]\n" "ldr x22, [x23], #0x8\n" - ".inst 0xe05b22a8 // ld1h { za1h.h[x13] }, p0/Z, [x21, x27, LSL #1]\n" - "add x13, x13, #0x2\n" + ".inst 0xe05902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28, LSL #1\n" "ldr x21, [x20], #0x8\n" - "cmp x13, x26, LSL #1\n" "blt 2b\n" "3:" // Loads: Tail - ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - "sub x20, %x[width], x28\n" + "sub x20, %x[width], x26\n" + ".inst 0x25286580 // psel p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05902c0 // ld1h { za0h.h[x12] }, p0/Z, [x22, x25, LSL #1]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + "cmp x20, x27\n" + ".inst 0xe05902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" "mov x12, #0x0\n" - "cmp x20, x25\n" - ".inst 0xe05b26c0 // ld1h { za0h.h[x13] }, p1/Z, [x22, x27, LSL #1]\n" - "csel x20, x20, x25, LT\n" - ".inst 0xe05b22a8 // ld1h { za1h.h[x13] }, p0/Z, [x21, x27, LSL #1]\n" + "csel x20, x20, x27, LT\n" "4:" // Stores: Loop - ".inst 0x25287541 // psel p1.h, p13.h/Z, p10.h[w12]\n" ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" - ".inst 0xe07f8700 // st1h { za0v.h[x12] }, p1/Z, [x24, XZR, LSL #1]\n" - ".inst 0xe0798308 // st1h { za1v.h[x12] }, p0/Z, [x24, x25, LSL #1]\n" + ".inst 0xe07f8300 // st1h { za0v.h[x12] }, p0/Z, [x24, XZR, LSL #1]\n" + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07b8308 // st1h { za1v.h[x12] }, p0/Z, [x24, x27, LSL #1]\n" "add x12, x12, #0x1\n" - "addvl x24, x24, #4\n" "cmp x12, x20\n" + "addvl x24, x24, #4\n" "blt 4b\n" - "inch x28\n" - "inch x27\n" - "whilelt p10.h, x28, %x[width]\n" - "whilelt p9.h, x28, %x[width]\n" - "whilelt p8.h, x28, %x[width]\n" + "inch x26\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "inch x25\n" "b.any 1b\n" "mov %x[out], x24\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp index 68fabe3523..b310651525 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp @@ -32,251 +32,256 @@ void interleave_block<2, 1, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" + "mov x22, %x[width]\n" + "incw x22\n" + "cntw x16\n" + "sub x22, x22, #0x1\n" + "udiv x22, x22, x16\n" // n_passes = ceildiv(width, VL<T>) "mov x21, %x[width]\n" - "mov x20, %x[width]\n" - "incw x21\n" - "cntw x17\n" - "sub x21, x21, #0x1\n" - "sub x16, x17, #0x1\n" - "udiv x21, x21, x17\n" // n_passes = ceildiv(width, VL<T>) - "ands x16, x20, x16\n" - "sub x20, x21, #0x1\n" - "sub x15, x17, #0x2\n" - "mov x14, #0x0\n" + "sub x15, x16, #0x1\n" + "sub x20, x22, #0x1\n" + "ands x15, x21, x15\n" + "sub x14, x16, #0x2\n" + "mov x13, #0x0\n" "mov x11, %x[in]\n" - "add x10, %x[in], x17, LSL #3\n" - "cntw x9, ALL, MUL #2\n" - "ldr x28, [x11, #0x0]\n" - "cntw x27, ALL, MUL #3\n" + "ldr x10, [x11, #0x0]\n" + "add x9, %x[in], x16, LSL #3\n" + "cntw x28, ALL, MUL #2\n" + "ldr x27, [x9, #0x0]\n" + "cntw x26, ALL, MUL #3\n" "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 - "ldr x26, [x10, #0x0]\n" - "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) - "csel x16, x16, x17, NE\n" - "ldr x24, [x11, #0x8]\n" - "ptrue p12.s\n" - "whilelt p11.s, XZR, %x[height]\n" - "ldr x21, [x10, #0x8]\n" - "whilelt p10.s, x17, %x[height]\n" + "ldr x25, [x11, #0x8]\n" + "and x24, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "csel x15, x15, x16, NE\n" + "ldr x21, [x9, #0x8]\n" + "ptrue p13.s\n" + "whilelt p12.s, XZR, %x[height]\n" + "whilelt p11.s, x16, %x[height]\n" "mov x23, %x[row_offset]\n" "mov x22, %x[out]\n" - "whilelt p9.s, x14, %x[width]\n" - "whilelt p8.s, x14, %x[width]\n" + "whilelt p10.s, x13, %x[width]\n" + "whilelt p9.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" "add x11, x11, #0x10\n" - "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" "mov x12, #0x0\n" - "cbz x15, 2f\n" + "cbz x14, 2f\n" "1:" // K loop: Charge: Loop - ".inst 0x25306163 // psel p3.s, p8.s/Z, p11.s[w12]\n" - ".inst 0x25306142 // psel p2.s, p8.s/Z, p10.s[w12]\n" - ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" - ".inst 0x25706140 // psel p0.s, p8.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0970f80 // ld1w { za0h.s[x12] }, p3/Z, [x28, x23, LSL #2]\n" - "ldr x28, [x11, #0x0]\n" - ".inst 0xe0970b44 // ld1w { za1h.s[x12] }, p2/Z, [x26, x23, LSL #2]\n" - "ldr x26, [x10, #0x0]\n" - ".inst 0xe0970701 // ld1w { za0h.s[x12, #1] }, p1/Z, [x24, x23, LSL #2]\n" - "ldr x24, [x11, #0x8]\n" + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + ".inst 0x25706581 // psel p1.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706160 // psel p0.s, p8.s/Z, p11.s[w12, #1]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0xe0970721 // ld1w { za0h.s[x12, #1] }, p1/Z, [x25, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" "add x11, x11, #0x10\n" ".inst 0xe09702a5 // ld1w { za1h.s[x12, #1] }, p0/Z, [x21, x23, LSL #2]\n" "add x12, x12, #0x2\n" - "ldr x21, [x10, #0x8]\n" - "add x10, x10, #0x10\n" - "cmp x12, x15\n" + "cmp x12, x14\n" + "ldr x21, [x9, #0x8]\n" + "add x9, x9, #0x10\n" "blt 1b\n" "2:" // K loop: Charge: End - ".inst 0x25306163 // psel p3.s, p8.s/Z, p11.s[w12]\n" - ".inst 0x25306142 // psel p2.s, p8.s/Z, p10.s[w12]\n" - ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" - ".inst 0x25706140 // psel p0.s, p8.s/Z, p10.s[w12, #1]\n" + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + ".inst 0x25706581 // psel p1.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706160 // psel p0.s, p8.s/Z, p11.s[w12, #1]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x17, LSL #3\n" - ".inst 0xe0970f80 // ld1w { za0h.s[x12] }, p3/Z, [x28, x23, LSL #2]\n" - "ldr x28, [x11, #0x0]\n" - "incw x14\n" - ".inst 0xe0970b44 // ld1w { za1h.s[x12] }, p2/Z, [x26, x23, LSL #2]\n" - "ldr x26, [x10, #0x0]\n" - ".inst 0xe0970701 // ld1w { za0h.s[x12, #1] }, p1/Z, [x24, x23, LSL #2]\n" - "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" + "add x9, %x[in], x16, LSL #3\n" + ".inst 0xe0970721 // ld1w { za0h.s[x12, #1] }, p1/Z, [x25, x23, LSL #2]\n" + "ldr x10, [x11, #0x0]\n" ".inst 0xe09702a5 // ld1w { za1h.s[x12, #1] }, p0/Z, [x21, x23, LSL #2]\n" - "ldr x21, [x10, #0x8]\n" - "add x10, x10, #0x10\n" + "ldr x27, [x9, #0x0]\n" "incw x23\n" + "incw x13\n" + "ldr x25, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + "ldr x21, [x9, #0x8]\n" + "add x9, x9, #0x10\n" "cbz x20, 8f\n" "mov x20, x20\n" "3:" // K loop: Main loop - "whilelt p8.s, x14, %x[width]\n" - "mov x13, #0x0\n" - "cbz x15, 5f\n" + "whilelt p9.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" + "mov x12, #0x0\n" + "cbz x14, 5f\n" "4:" // K loop: Main loop: First: Loop - ".inst 0x25316160 // psel p0.s, p8.s/Z, p11.s[w13]\n" - ".inst 0x25316142 // psel p2.s, p8.s/Z, p10.s[w13]\n" - ".inst 0x25716161 // psel p1.s, p8.s/Z, p11.s[w13, #1]\n" - ".inst 0x25716143 // psel p3.s, p8.s/Z, p10.s[w13, #1]\n" - ".inst 0xe0972388 // ld1w { za2h.s[x13] }, p0/Z, [x28, x23, LSL #2]\n" - ".inst 0x25317120 // psel p0.s, p12.s/Z, p9.s[w13]\n" - "ldr x28, [x11, #0x0]\n" - ".inst 0xe0972b4c // ld1w { za3h.s[x13] }, p2/Z, [x26, x23, LSL #2]\n" - ".inst 0x25317122 // psel p2.s, p12.s/Z, p9.s[w13]\n" - "ldr x26, [x10, #0x0]\n" - ".inst 0xe0972709 // ld1w { za2h.s[x13, #1] }, p1/Z, [x24, x23, LSL #2]\n" - ".inst 0x25717121 // psel p1.s, p12.s/Z, p9.s[w13, #1]\n" - "ldr x24, [x11, #0x8]\n" + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970548 // ld1w { za2h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0xe097036c // ld1w { za3h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706162 // psel p2.s, p8.s/Z, p11.s[w12, #1]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0970329 // ld1w { za2h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0970aad // ld1w { za3h.s[x12, #1] }, p2/Z, [x21, x23, LSL #2]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe0972ead // ld1w { za3h.s[x13, #1] }, p3/Z, [x21, x23, LSL #2]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0xe0bfa2c0 // st1w { za0v.s[x13] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x25717120 // psel p0.s, p12.s/Z, p9.s[w13, #1]\n" - ".inst 0xe0b1aac4 // st1w { za1v.s[x13] }, p2/Z, [x22, x17, LSL #2]\n" - "add x10, x10, #0x10\n" - ".inst 0xe0a9a6c1 // st1w { za0v.s[x13, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bba2c5 // st1w { za1v.s[x13, #1] }, p0/Z, [x22, x27, LSL #2]\n" - "add x13, x13, #0x2\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x9, x9, #0x10\n" + ".inst 0xe0ba82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x14\n" "addvl x22, x22, #4\n" - "cmp x13, x15\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail - ".inst 0x25316160 // psel p0.s, p8.s/Z, p11.s[w13]\n" - ".inst 0x25316142 // psel p2.s, p8.s/Z, p10.s[w13]\n" - ".inst 0x25716161 // psel p1.s, p8.s/Z, p11.s[w13, #1]\n" - ".inst 0x25716143 // psel p3.s, p8.s/Z, p10.s[w13, #1]\n" + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970548 // ld1w { za2h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + ".inst 0xe097036c // ld1w { za3h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x17, LSL #3\n" - ".inst 0xe0972388 // ld1w { za2h.s[x13] }, p0/Z, [x28, x23, LSL #2]\n" - ".inst 0x25317120 // psel p0.s, p12.s/Z, p9.s[w13]\n" - "ldr x28, [x11, #0x0]\n" - "mov x12, #0x0\n" - ".inst 0xe0972b4c // ld1w { za3h.s[x13] }, p2/Z, [x26, x23, LSL #2]\n" - ".inst 0x25317122 // psel p2.s, p12.s/Z, p9.s[w13]\n" - "ldr x26, [x10, #0x0]\n" - ".inst 0xe0972709 // ld1w { za2h.s[x13, #1] }, p1/Z, [x24, x23, LSL #2]\n" - ".inst 0x25717121 // psel p1.s, p12.s/Z, p9.s[w13, #1]\n" - "ldr x24, [x11, #0x8]\n" + "add x9, %x[in], x16, LSL #3\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe0970329 // ld1w { za2h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe09706ad // ld1w { za3h.s[x12, #1] }, p1/Z, [x21, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.s, x13, %x[width]\n" + "incw x13\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x11, x11, #0x10\n" - ".inst 0xe0972ead // ld1w { za3h.s[x13, #1] }, p3/Z, [x21, x23, LSL #2]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0xe0bfa2c0 // st1w { za0v.s[x13] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x25717120 // psel p0.s, p12.s/Z, p9.s[w13, #1]\n" - ".inst 0xe0b1aac4 // st1w { za1v.s[x13] }, p2/Z, [x22, x17, LSL #2]\n" - "whilelt p9.s, x14, %x[width]\n" - "incw x14\n" - ".inst 0xe0a9a6c1 // st1w { za0v.s[x13, #1] }, p1/Z, [x22, x9, LSL #2]\n" - "add x10, x10, #0x10\n" - "incw x23\n" - ".inst 0xe0bba2c5 // st1w { za1v.s[x13, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "add x9, x9, #0x10\n" + ".inst 0xe0ba82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" "addvl x22, x22, #4\n" - "whilelt p8.s, x14, %x[width]\n" - "cbz x15, 7f\n" + "incw x23\n" + "whilelt p9.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" + "mov x12, #0x0\n" + "cbz x14, 7f\n" "6:" // K loop: Main loop: Second: Loop + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" - ".inst 0x25306142 // psel p2.s, p8.s/Z, p10.s[w12]\n" - ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" - ".inst 0x25706143 // psel p3.s, p8.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0970380 // ld1w { za0h.s[x12] }, p0/Z, [x28, x23, LSL #2]\n" - ".inst 0x25307120 // psel p0.s, p12.s/Z, p9.s[w12]\n" - "ldr x28, [x11, #0x0]\n" - ".inst 0xe0970b44 // ld1w { za1h.s[x12] }, p2/Z, [x26, x23, LSL #2]\n" - ".inst 0x25307122 // psel p2.s, p12.s/Z, p9.s[w12]\n" - "ldr x26, [x10, #0x0]\n" - ".inst 0xe0970701 // ld1w { za0h.s[x12, #1] }, p1/Z, [x24, x23, LSL #2]\n" - ".inst 0x25707121 // psel p1.s, p12.s/Z, p9.s[w12, #1]\n" - "ldr x24, [x11, #0x8]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706162 // psel p2.s, p8.s/Z, p11.s[w12, #1]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0970321 // ld1w { za0h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0970aa5 // ld1w { za1h.s[x12, #1] }, p2/Z, [x21, x23, LSL #2]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe0970ea5 // ld1w { za1h.s[x12, #1] }, p3/Z, [x21, x23, LSL #2]\n" - "ldr x21, [x10, #0x8]\n" - ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x25707120 // psel p0.s, p12.s/Z, p9.s[w12, #1]\n" - ".inst 0xe0b18acc // st1w { za3v.s[x12] }, p2/Z, [x22, x17, LSL #2]\n" - "add x10, x10, #0x10\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" - ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x9, x9, #0x10\n" + ".inst 0xe0ba82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" "add x12, x12, #0x2\n" + "cmp x12, x14\n" "addvl x22, x22, #4\n" - "cmp x12, x15\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" - ".inst 0x25306142 // psel p2.s, p8.s/Z, p10.s[w12]\n" - ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" - ".inst 0x25706143 // psel p3.s, p8.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" "mov x11, %x[in]\n" - "add x10, %x[in], x17, LSL #3\n" - ".inst 0xe0970380 // ld1w { za0h.s[x12] }, p0/Z, [x28, x23, LSL #2]\n" - ".inst 0x25307120 // psel p0.s, p12.s/Z, p9.s[w12]\n" - "ldr x28, [x11, #0x0]\n" - ".inst 0xe0970b44 // ld1w { za1h.s[x12] }, p2/Z, [x26, x23, LSL #2]\n" - ".inst 0x25307122 // psel p2.s, p12.s/Z, p9.s[w12]\n" - "ldr x26, [x10, #0x0]\n" - ".inst 0xe0970701 // ld1w { za0h.s[x12, #1] }, p1/Z, [x24, x23, LSL #2]\n" - ".inst 0x25707121 // psel p1.s, p12.s/Z, p9.s[w12, #1]\n" - "ldr x24, [x11, #0x8]\n" - "add x11, x11, #0x10\n" - ".inst 0xe0970ea5 // ld1w { za1h.s[x12, #1] }, p3/Z, [x21, x23, LSL #2]\n" - "ldr x21, [x10, #0x8]\n" + "add x9, %x[in], x16, LSL #3\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe0970321 // ld1w { za0h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe09706a5 // ld1w { za1h.s[x12, #1] }, p1/Z, [x21, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" + "ldr x21, [x9, #0x8]\n" ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" - ".inst 0x25707120 // psel p0.s, p12.s/Z, p9.s[w12, #1]\n" - ".inst 0xe0b18acc // st1w { za3v.s[x12] }, p2/Z, [x22, x17, LSL #2]\n" - "whilelt p9.s, x14, %x[width]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08acc // st1w { za3v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.s, x13, %x[width]\n" "subs x20, x20, #0x1\n" - ".inst 0xe0a986c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x9, LSL #2]\n" - "add x10, x10, #0x10\n" - "incw x14\n" - ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x9, x9, #0x10\n" + ".inst 0xe0ba82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" "addvl x22, x22, #4\n" + "incw x13\n" "incw x23\n" "bgt 3b\n" "8:" // K loop: Tails - "cbnz x25, 11f\n" + "cbnz x24, 11f\n" "mov x11, %x[in]\n" - "whilelt p8.s, x14, %x[width]\n" + "whilelt p9.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First - ".inst 0x25307123 // psel p3.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25307122 // psel p2.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25306161 // psel p1.s, p8.s/Z, p11.s[w12]\n" - ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" - ".inst 0xe0bf8ec0 // st1w { za0v.s[x12] }, p3/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0b18ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x17, LSL #2]\n" - "addvl x22, x22, #2\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "ldr x21, [x11, #0x0]\n" - "ldr x20, [x11, x17, LSL #0x3]\n" - "add x11, x11, #0x8\n" + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" ".inst 0xe09706a8 // ld1w { za2h.s[x12] }, p1/Z, [x21, x23, LSL #2]\n" + "add x11, x11, #0x8\n" + "addvl x22, x22, #2\n" ".inst 0xe097028c // ld1w { za3h.s[x12] }, p0/Z, [x20, x23, LSL #2]\n" "add x12, x12, #0x1\n" - "cmp x12, x17\n" + "cmp x12, x16\n" "blt 9b\n" - "whilelt p9.s, x14, %x[width]\n" - "whilelt p8.s, x14, %x[width]\n" + "whilelt p10.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second - ".inst 0x25307121 // psel p1.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25307120 // psel p0.s, p12.s/Z, p9.s[w12]\n" - ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0b182cc // st1w { za3v.s[x12] }, p0/Z, [x22, x17, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" + "cmp x12, x15\n" "addvl x22, x22, #2\n" - "cmp x12, x16\n" "blt 10b\n" - "whilelt p8.s, x14, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" "12:" // K loop: Tails: Odd: Loop - ".inst 0x25307121 // psel p1.s, p12.s/Z, p9.s[w12]\n" - ".inst 0x25307120 // psel p0.s, p12.s/Z, p9.s[w12]\n" - ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" - ".inst 0xe0b182c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x17, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" + "cmp x12, x15\n" "addvl x22, x22, #2\n" - "cmp x12, x16\n" "blt 12b\n" "13:" // K loop: End "mov %x[out], x22\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp index f5c756eba6..6903945536 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp @@ -32,93 +32,93 @@ void interleave_block<4, 2, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x17, #0x0\n" - "mov x16, %x[row_offset]\n" + "cntw x16\n" "cntw x15\n" - "cntw x14\n" - "cntw x11, ALL, MUL #2\n" - "cntw x10, ALL, MUL #3\n" - "cmp %x[height], x15\n" - "cnth x9\n" - "csel x15, %x[height], x15, LT\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" "whilelt p11.h, XZR, %x[height]\n" - "whilelt p10.h, x14, %x[height]\n" - "whilelt p9.h, x11, %x[height]\n" - "whilelt p8.h, x10, %x[height]\n" + "whilelt p10.h, x15, %x[height]\n" + "whilelt p9.h, x14, %x[height]\n" + "whilelt p8.h, x13, %x[height]\n" + "mov x11, #0x0\n" + "cnth x10\n" "ptrue p13.s\n" - "sub x15, x15, #0x1\n" + "sub x16, x16, #0x1\n" "zip1 p12.h, p11.h, p9.h\n" "zip1 p11.h, p10.h, p8.h\n" + "mov x9, %x[row_offset]\n" "mov x28, %x[out]\n" - "whilelt p10.h, x17, %x[width]\n" - "whilelt p9.h, x17, %x[width]\n" - "whilelt p8.h, x17, %x[width]\n" + "whilelt p10.h, x11, %x[width]\n" + "whilelt p9.h, x11, %x[width]\n" + "whilelt p8.h, x11, %x[width]\n" "1:" // Width loop "add x27, %x[in], XZR, LSL #3\n" - "add x26, %x[in], x14, LSL #3\n" - "add x25, %x[in], x11, LSL #3\n" - "add x20, %x[in], x10, LSL #3\n" - "ldr x24, [x27], #0x8\n" - "mov x13, #0x0\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x20, %x[in], x13, LSL #3\n" "ldr x23, [x26], #0x8\n" - "ldr x22, [x25], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" "ldr x21, [x20], #0x8\n" - "cbz x15, 3f\n" + "cbz x16, 3f\n" "2:" // Loads: Loop - ".inst 0x25296580 // psel p0.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296162 // psel p2.h, p8.h/Z, p11.h[w13]\n" - ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" - ".inst 0xe0502300 // ld1h { za0h.h[x13] }, p0/Z, [x24, x16, LSL #1]\n" - ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" - "ldr x24, [x27], #0x8\n" - ".inst 0xe0502ae8 // ld1h { za1h.h[x13] }, p2/Z, [x23, x16, LSL #1]\n" + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0490720 // ld1h { za0h.h[x12] }, p1/Z, [x25, x9, LSL #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe04902e8 // ld1h { za1h.h[x12] }, p0/Z, [x23, x9, LSL #1]\n" + ".inst 0x25386581 // psel p1.h, p9.h/Z, p12.h[w12, #1]\n" + ".inst 0x25386160 // psel p0.h, p8.h/Z, p11.h[w12, #1]\n" "ldr x23, [x26], #0x8\n" - ".inst 0xe05026c1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x22, x16, LSL #1]\n" - "ldr x22, [x25], #0x8\n" - ".inst 0xe05022a9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x21, x16, LSL #1]\n" - "add x13, x13, #0x2\n" + ".inst 0xe04906c1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x22, x9, LSL #1]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe04902a9 // ld1h { za1h.h[x12, #1] }, p0/Z, [x21, x9, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x16, LSL #1\n" "ldr x21, [x20], #0x8\n" - "cmp x13, x15, LSL #1\n" "blt 2b\n" "3:" // Loads: Tail - ".inst 0x25296580 // psel p0.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296162 // psel p2.h, p8.h/Z, p11.h[w13]\n" - "sub x20, %x[width], x17\n" - ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" - "cmp x20, x9\n" - "mov x12, #0x0\n" - ".inst 0xe0502300 // ld1h { za0h.h[x13] }, p0/Z, [x24, x16, LSL #1]\n" - ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" - "csel x20, x20, x9, LT\n" - ".inst 0xe0502ae8 // ld1h { za1h.h[x13] }, p2/Z, [x23, x16, LSL #1]\n" + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0490720 // ld1h { za0h.h[x12] }, p1/Z, [x25, x9, LSL #1]\n" + "sub x20, %x[width], x11\n" + ".inst 0xe04902e8 // ld1h { za1h.h[x12] }, p0/Z, [x23, x9, LSL #1]\n" + "cmp x20, x10\n" + "csel x20, x20, x10, LT\n" + ".inst 0x25386580 // psel p0.h, p9.h/Z, p12.h[w12, #1]\n" + ".inst 0xe04902c1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x22, x9, LSL #1]\n" + ".inst 0x25386160 // psel p0.h, p8.h/Z, p11.h[w12, #1]\n" "add x20, x20, #0x1\n" - ".inst 0xe05026c1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x22, x16, LSL #1]\n" + ".inst 0xe04902a9 // ld1h { za1h.h[x12, #1] }, p0/Z, [x21, x9, LSL #1]\n" + "mov x12, #0x0\n" "lsr x20, x20, #0x1\n" - ".inst 0xe05022a9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x21, x16, LSL #1]\n" "4:" // Stores: Loop ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" - ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0ae8b84 // st1w { za1v.s[x12] }, p2/Z, [x28, x14, LSL #2]\n" - ".inst 0xe0ab8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x11, LSL #2]\n" - ".inst 0xe0aa838c // st1w { za3v.s[x12] }, p0/Z, [x28, x10, LSL #2]\n" + ".inst 0xe0af8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x15, LSL #2]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0ae8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x14, LSL #2]\n" + ".inst 0xe0ad838c // st1w { za3v.s[x12] }, p0/Z, [x28, x13, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x28, x28, #4\n" "cmp x12, x20\n" + "addvl x28, x28, #4\n" "blt 4b\n" - "inch x17\n" - "inch x16\n" - "whilelt p10.h, x17, %x[width]\n" - "whilelt p9.h, x17, %x[width]\n" - "whilelt p8.h, x17, %x[width]\n" + "inch x11\n" + "whilelt p10.h, x11, %x[width]\n" + "whilelt p9.h, x11, %x[width]\n" + "whilelt p8.h, x11, %x[width]\n" + "inch x9\n" "b.any 1b\n" "mov %x[out], x28\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp index 9e0ab463be..b1ba9a4fe7 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME) +#if defined(__ARM_FEATURE_SVE) template <> void interleave_block<4, 2, VLType::SME, false>( @@ -65,36 +65,36 @@ void interleave_block<4, 2, VLType::SME, false>( "ldr x21, [x20], #0x8\n" "cbz x15, 3f\n" "2:" // Loads: Loop - ".inst 0x25296580 // psel p0.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296162 // psel p2.h, p8.h/Z, p11.h[w13]\n" - ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" - ".inst 0xe0502300 // ld1h { za0h.h[x13] }, p0/Z, [x24, x16, LSL #1]\n" - ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0x25296582 // psel p2.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296161 // psel p1.h, p8.h/Z, p11.h[w13]\n" + ".inst 0x25396580 // psel p0.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0xe0502b00 // ld1h { za0h.h[x13] }, p2/Z, [x24, x16, LSL #1]\n" + ".inst 0x25396162 // psel p2.h, p8.h/Z, p11.h[w13, #1]\n" "ldr x24, [x27], #0x8\n" - ".inst 0xe0502ae8 // ld1h { za1h.h[x13] }, p2/Z, [x23, x16, LSL #1]\n" + ".inst 0xe05026e8 // ld1h { za1h.h[x13] }, p1/Z, [x23, x16, LSL #1]\n" "ldr x23, [x26], #0x8\n" - ".inst 0xe05026c1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x22, x16, LSL #1]\n" + ".inst 0xe05022c1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x22, x16, LSL #1]\n" "ldr x22, [x25], #0x8\n" - ".inst 0xe05022a9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x21, x16, LSL #1]\n" + ".inst 0xe0502aa9 // ld1h { za1h.h[x13, #1] }, p2/Z, [x21, x16, LSL #1]\n" "add x13, x13, #0x2\n" "ldr x21, [x20], #0x8\n" "cmp x13, x15, LSL #1\n" "blt 2b\n" "3:" // Loads: Tail - ".inst 0x25296580 // psel p0.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296162 // psel p2.h, p8.h/Z, p11.h[w13]\n" + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" "sub x20, %x[width], x17\n" - ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396582 // psel p2.h, p9.h/Z, p12.h[w13, #1]\n" "cmp x20, x9\n" "mov x12, #0x0\n" - ".inst 0xe0502300 // ld1h { za0h.h[x13] }, p0/Z, [x24, x16, LSL #1]\n" - ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0502700 // ld1h { za0h.h[x13] }, p1/Z, [x24, x16, LSL #1]\n" + ".inst 0xe05022e8 // ld1h { za1h.h[x13] }, p0/Z, [x23, x16, LSL #1]\n" + ".inst 0x25396161 // psel p1.h, p8.h/Z, p11.h[w13, #1]\n" "csel x20, x20, x9, LT\n" - ".inst 0xe0502ae8 // ld1h { za1h.h[x13] }, p2/Z, [x23, x16, LSL #1]\n" "add x20, x20, #0x1\n" - ".inst 0xe05026c1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x22, x16, LSL #1]\n" + ".inst 0xe0502ac1 // ld1h { za0h.h[x13, #1] }, p2/Z, [x22, x16, LSL #1]\n" "lsr x20, x20, #0x1\n" - ".inst 0xe05022a9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x21, x16, LSL #1]\n" + ".inst 0xe05026a9 // ld1h { za1h.h[x13, #1] }, p1/Z, [x21, x16, LSL #1]\n" "4:" // Stores: Loop ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" @@ -122,4 +122,4 @@ void interleave_block<4, 2, VLType::SME, false>( ); } -#endif // defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp index b0b3aa85c1..9a5521aa8a 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp @@ -32,92 +32,92 @@ void interleave_block<4, 4, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x17, #0x0\n" - "mov x16, %x[row_offset]\n" + "cntw x16\n" "cntw x15\n" - "cntw x14\n" - "cntw x11, ALL, MUL #2\n" - "cntw x10, ALL, MUL #3\n" - "cmp %x[height], x15\n" - "cntb x9\n" - "csel x15, %x[height], x15, LT\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" "whilelt p12.b, XZR, %x[height]\n" - "whilelt p10.b, x14, %x[height]\n" - "whilelt p9.b, x11, %x[height]\n" - "whilelt p8.b, x10, %x[height]\n" + "whilelt p10.b, x15, %x[height]\n" + "whilelt p9.b, x14, %x[height]\n" + "whilelt p8.b, x13, %x[height]\n" "zip1 p12.b, p12.b, p9.b\n" "zip1 p10.b, p10.b, p8.b\n" + "mov x11, #0x0\n" + "cntb x10\n" "ptrue p11.s\n" - "sub x15, x15, #0x1\n" + "sub x16, x16, #0x1\n" "zip1 p10.b, p12.b, p10.b\n" + "mov x9, %x[row_offset]\n" "mov x28, %x[out]\n" - "whilelt p9.b, x17, %x[width]\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p9.b, x11, %x[width]\n" + "whilelt p8.b, x11, %x[width]\n" "1:" // Width loop "add x27, %x[in], XZR, LSL #3\n" - "add x26, %x[in], x14, LSL #3\n" - "add x25, %x[in], x11, LSL #3\n" - "add x20, %x[in], x10, LSL #3\n" - "ldr x24, [x27], #0x8\n" - "mov x13, #0x0\n" - "ldr x23, [x26], #0x8\n" - "ldr x22, [x25], #0x8\n" - "ldr x21, [x20], #0x8\n" - "cbz x15, 3f\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x23, %x[in], x13, LSL #3\n" + "ldr x20, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x23], #0x8\n" + "cbz x16, 3f\n" "2:" // Loads: Loop - ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xe0102300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x16]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - "ldr x24, [x27], #0x8\n" - ".inst 0xe0102ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x16]\n" - "ldr x23, [x26], #0x8\n" - ".inst 0xe01026c2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x22, x16]\n" - "ldr x22, [x25], #0x8\n" - ".inst 0xe01022a3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x21, x16]\n" - "add x13, x13, #0x4\n" - "ldr x21, [x20], #0x8\n" - "cmp x13, x15, LSL #2\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0090320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x9]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe0090281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x9]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0xe00906c2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x22, x9]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe00902a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x9]\n" + "add x12, x12, #0x4\n" + "cmp x12, x16, LSL #2\n" + "ldr x21, [x23], #0x8\n" "blt 2b\n" "3:" // Loads: Tail - ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - "sub x20, %x[width], x17\n" - "cmp x20, x9\n" - "mov x12, #0x0\n" - ".inst 0xe0102300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x16]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - "csel x20, x20, x9, LT\n" - ".inst 0xe0102ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x16]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0090320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x9]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe0090281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x9]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" + "sub x20, %x[width], x11\n" + ".inst 0xe00902c2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x22, x9]\n" + "cmp x20, x10\n" + "csel x20, x20, x10, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" "add x20, x20, #0x3\n" - ".inst 0xe01026c2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x22, x16]\n" + ".inst 0xe00902a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x9]\n" + "mov x12, #0x0\n" "lsr x20, x20, #0x2\n" - ".inst 0xe01022a3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x21, x16]\n" "4:" // Stores: Loop ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xe0ae8b84 // st1w { za1v.s[x12] }, p2/Z, [x28, x14, LSL #2]\n" - ".inst 0xe0ab8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x11, LSL #2]\n" - ".inst 0xe0aa838c // st1w { za3v.s[x12] }, p0/Z, [x28, x10, LSL #2]\n" + ".inst 0xe0af8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x15, LSL #2]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0ae8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x14, LSL #2]\n" + ".inst 0xe0ad838c // st1w { za3v.s[x12] }, p0/Z, [x28, x13, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x28, x28, #4\n" "cmp x12, x20\n" + "addvl x28, x28, #4\n" "blt 4b\n" - "incb x17\n" - "incb x16\n" - "whilelt p9.b, x17, %x[width]\n" - "whilelt p8.b, x17, %x[width]\n" + "incb x11\n" + "whilelt p9.b, x11, %x[width]\n" + "whilelt p8.b, x11, %x[width]\n" + "incb x9\n" "b.any 1b\n" "mov %x[out], x28\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp index a4696816e9..22ec1011cc 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp @@ -32,118 +32,118 @@ void interleave_block<4, 4, VLType::SME, true>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x17, %x[row_offset]\n" - "mov x16, %x[out]\n" + "cntw x16\n" "cntw x15\n" - "cntw x14\n" "mov z24.b, #0x1\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" "mov z23.s, #0x0\n" - "cntw x11, ALL, MUL #2\n" - "cntw x10, ALL, MUL #3\n" "mov z22.s, #0x0\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" "mov z21.s, #0x0\n" - "cmp %x[height], x15\n" - "ptrue p3.b\n" "mov z20.s, #0x0\n" - "csel x15, %x[height], x15, LT\n" "whilelt p12.b, XZR, %x[height]\n" - "whilelt p10.b, x14, %x[height]\n" - "whilelt p9.b, x11, %x[height]\n" - "whilelt p8.b, x10, %x[height]\n" + "whilelt p10.b, x15, %x[height]\n" + "whilelt p9.b, x14, %x[height]\n" + "whilelt p8.b, x13, %x[height]\n" "zip1 p12.b, p12.b, p9.b\n" "zip1 p10.b, p10.b, p8.b\n" - "cntb x9\n" + "ptrue p2.b\n" + "cntb x11\n" "ptrue p11.s\n" - "sub x15, x15, #0x1\n" + "sub x16, x16, #0x1\n" "zip1 p10.b, p12.b, p10.b\n" + "mov x10, %x[row_offset]\n" + "mov x9, %x[out]\n" "cbnz %x[first], 1f\n" - "addvl x16, x16, #-4\n" - "ld1w { z23.s }, p3/Z, [x16]\n" - "ld1w { z22.s }, p3/Z, [x16, #1, MUL VL]\n" - "ld1w { z21.s }, p3/Z, [x16, #2, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x16, #3, MUL VL]\n" + "addvl x9, x9, #-4\n" + "ld1w { z23.s }, p2/Z, [x9]\n" + "ld1w { z22.s }, p2/Z, [x9, #1, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x9, #3, MUL VL]\n" "1:" // Initialise row sums: End "mov x28, #0x0\n" "whilelt p9.b, x28, %x[width]\n" "whilelt p8.b, x28, %x[width]\n" "2:" // Width loop "add x27, %x[in], XZR, LSL #3\n" - "add x26, %x[in], x14, LSL #3\n" - "add x25, %x[in], x11, LSL #3\n" - "add x20, %x[in], x10, LSL #3\n" - "ldr x24, [x27], #0x8\n" - "mov x13, #0x0\n" - "ldr x23, [x26], #0x8\n" - "ldr x22, [x25], #0x8\n" - "ldr x21, [x20], #0x8\n" - "cbz x15, 4f\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x23, %x[in], x13, LSL #3\n" + "ldr x20, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x23], #0x8\n" + "cbz x16, 4f\n" "3:" // Loads: Loop - ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xe0112300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x17]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - "ldr x24, [x27], #0x8\n" - ".inst 0xe0112ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x17]\n" - "ldr x23, [x26], #0x8\n" - ".inst 0xe01126c2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x22, x17]\n" - "ldr x22, [x25], #0x8\n" - ".inst 0xe01122a3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x21, x17]\n" - "add x13, x13, #0x4\n" - "ldr x21, [x20], #0x8\n" - "cmp x13, x15, LSL #2\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe00a0320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x10]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe00a0281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x10]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0xe00a06c2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x22, x10]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe00a02a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x10]\n" + "add x12, x12, #0x4\n" + "cmp x12, x16, LSL #2\n" + "ldr x21, [x23], #0x8\n" "blt 3b\n" "4:" // Loads: Tail - ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe00a0320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x10]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe00a0281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x10]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" "sub x20, %x[width], x28\n" - "cmp x20, x9\n" - "mov x12, #0x0\n" - ".inst 0xe0112300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x17]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - "csel x20, x20, x9, LT\n" - ".inst 0xe0112ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x17]\n" + ".inst 0xe00a02c2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x22, x10]\n" + "cmp x20, x11\n" + "csel x20, x20, x11, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" "add x20, x20, #0x3\n" - ".inst 0xe01126c2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x22, x17]\n" + ".inst 0xe00a02a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x10]\n" + "mov x12, #0x0\n" "lsr x20, x20, #0x2\n" - ".inst 0xe01122a3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x21, x17]\n" "5:" // Stores: Loop ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0828c11 // mova z17.s, p3/M, za0v.s[x12]\n" + ".inst 0xe0bf8120 // st1w { za0v.s[x12] }, p0/Z, [x9, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0xe0af8124 // st1w { za1v.s[x12] }, p0/Z, [x9, x15, LSL #2]\n" ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0828c90 // mova z16.s, p3/M, za1v.s[x12]\n" - "sdot z23.s, z17.b, z24.b\n" - ".inst 0xc0828d13 // mova z19.s, p3/M, za2v.s[x12]\n" - "sdot z22.s, z16.b, z24.b\n" - ".inst 0xe0bf8200 // st1w { za0v.s[x12] }, p0/Z, [x16, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0828d92 // mova z18.s, p3/M, za3v.s[x12]\n" - "sdot z21.s, z19.b, z24.b\n" - ".inst 0xe0ae8a04 // st1w { za1v.s[x12] }, p2/Z, [x16, x14, LSL #2]\n" - "sdot z20.s, z18.b, z24.b\n" - ".inst 0xe0ab8608 // st1w { za2v.s[x12] }, p1/Z, [x16, x11, LSL #2]\n" - ".inst 0xe0aa820c // st1w { za3v.s[x12] }, p0/Z, [x16, x10, LSL #2]\n" + ".inst 0xc0828893 // mova z19.s, p2/M, za1v.s[x12]\n" + ".inst 0xe0ae8528 // st1w { za2v.s[x12] }, p1/Z, [x9, x14, LSL #2]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + "sdot z23.s, z17.b, z24.b\n" + ".inst 0xe0ad812c // st1w { za3v.s[x12] }, p0/Z, [x9, x13, LSL #2]\n" + ".inst 0xc0828992 // mova z18.s, p2/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" - "addvl x16, x16, #4\n" "cmp x12, x20\n" + "sdot z22.s, z19.b, z24.b\n" + "sdot z21.s, z16.b, z24.b\n" + "addvl x9, x9, #4\n" + "sdot z20.s, z18.b, z24.b\n" "blt 5b\n" "incb x28\n" - "incb x17\n" "whilelt p9.b, x28, %x[width]\n" "whilelt p8.b, x28, %x[width]\n" + "incb x10\n" "b.any 2b\n" - "st1w { z23.s }, p3, [x16]\n" - "st1w { z22.s }, p3, [x16, #1, MUL VL]\n" - "st1w { z21.s }, p3, [x16, #2, MUL VL]\n" - "st1w { z20.s }, p3, [x16, #3, MUL VL]\n" - "addvl x16, x16, #4\n" + "st1w { z23.s }, p2, [x9]\n" + "st1w { z22.s }, p2, [x9, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x9, #2, MUL VL]\n" + "st1w { z20.s }, p2, [x9, #3, MUL VL]\n" + "addvl x9, x9, #4\n" + "mov %x[out], x9\n" ".inst 0xd503467f // SMSTOP\n" - "mov %x[out], x16\n" : [out] "+&r" (out) : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp index df77398acc..a99c2ea91b 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp @@ -32,92 +32,92 @@ void interleave_block<4, 4, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x17, #0x0\n" - "mov x16, %x[row_offset]\n" + "cntw x16\n" "cntw x15\n" - "cntw x14\n" - "cntw x11, ALL, MUL #2\n" - "cntw x10, ALL, MUL #3\n" - "cmp %x[height], x15\n" - "cntb x9\n" - "csel x15, %x[height], x15, LT\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" "whilelt p12.b, XZR, %x[height]\n" - "whilelt p10.b, x14, %x[height]\n" - "whilelt p9.b, x11, %x[height]\n" - "whilelt p8.b, x10, %x[height]\n" + "whilelt p10.b, x15, %x[height]\n" + "whilelt p9.b, x14, %x[height]\n" + "whilelt p8.b, x13, %x[height]\n" "zip1 p12.b, p12.b, p9.b\n" "zip1 p10.b, p10.b, p8.b\n" + "mov x11, #0x0\n" + "cntb x10\n" "ptrue p11.s\n" - "sub x15, x15, #0x1\n" + "sub x16, x16, #0x1\n" "zip1 p10.b, p12.b, p10.b\n" + "mov x9, %x[row_offset]\n" "mov x28, %x[out]\n" - "whilelt p9.b, x17, %x[width]\n" - "whilelt p8.b, x17, %x[width]\n" + "whilelt p9.b, x11, %x[width]\n" + "whilelt p8.b, x11, %x[width]\n" "1:" // Width loop "add x27, %x[in], XZR, LSL #3\n" - "add x26, %x[in], x14, LSL #3\n" - "add x25, %x[in], x11, LSL #3\n" - "add x20, %x[in], x10, LSL #3\n" - "ldr x24, [x27], #0x8\n" - "mov x13, #0x0\n" - "ldr x23, [x26], #0x8\n" - "ldr x22, [x25], #0x8\n" - "ldr x21, [x20], #0x8\n" - "cbz x15, 3f\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x23, %x[in], x13, LSL #3\n" + "ldr x20, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x23], #0x8\n" + "cbz x16, 3f\n" "2:" // Loads: Loop - ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xe0102300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x16]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - "ldr x24, [x27], #0x8\n" - ".inst 0xe0102ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x16]\n" - "ldr x23, [x26], #0x8\n" - ".inst 0xe01026c2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x22, x16]\n" - "ldr x22, [x25], #0x8\n" - ".inst 0xe01022a3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x21, x16]\n" - "add x13, x13, #0x4\n" - "ldr x21, [x20], #0x8\n" - "cmp x13, x15, LSL #2\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0090320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x9]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe0090281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x9]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0xe00906c2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x22, x9]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe00902a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x9]\n" + "add x12, x12, #0x4\n" + "cmp x12, x16, LSL #2\n" + "ldr x21, [x23], #0x8\n" "blt 2b\n" "3:" // Loads: Tail - ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - "sub x20, %x[width], x17\n" - "cmp x20, x9\n" - "mov x12, #0x0\n" - ".inst 0xe0102300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x16]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - "csel x20, x20, x9, LT\n" - ".inst 0xe0102ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x16]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0090320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x9]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe0090281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x9]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" + "sub x20, %x[width], x11\n" + ".inst 0xe00902c2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x22, x9]\n" + "cmp x20, x10\n" + "csel x20, x20, x10, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" "add x20, x20, #0x3\n" - ".inst 0xe01026c2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x22, x16]\n" + ".inst 0xe00902a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x9]\n" + "mov x12, #0x0\n" "lsr x20, x20, #0x2\n" - ".inst 0xe01022a3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x21, x16]\n" "4:" // Stores: Loop ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xe0ae8b84 // st1w { za1v.s[x12] }, p2/Z, [x28, x14, LSL #2]\n" - ".inst 0xe0ab8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x11, LSL #2]\n" - ".inst 0xe0aa838c // st1w { za3v.s[x12] }, p0/Z, [x28, x10, LSL #2]\n" + ".inst 0xe0af8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x15, LSL #2]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0ae8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x14, LSL #2]\n" + ".inst 0xe0ad838c // st1w { za3v.s[x12] }, p0/Z, [x28, x13, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x28, x28, #4\n" "cmp x12, x20\n" + "addvl x28, x28, #4\n" "blt 4b\n" - "incb x17\n" - "incb x16\n" - "whilelt p9.b, x17, %x[width]\n" - "whilelt p8.b, x17, %x[width]\n" + "incb x11\n" + "whilelt p9.b, x11, %x[width]\n" + "whilelt p8.b, x11, %x[width]\n" + "incb x9\n" "b.any 1b\n" "mov %x[out], x28\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp index 14ab3f476b..0a826d38f2 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp @@ -32,118 +32,118 @@ void interleave_block<4, 4, VLType::SME, true>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x17, %x[row_offset]\n" - "mov x16, %x[out]\n" + "cntw x16\n" "cntw x15\n" - "cntw x14\n" "mov z24.b, #0x1\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" "mov z23.s, #0x0\n" - "cntw x11, ALL, MUL #2\n" - "cntw x10, ALL, MUL #3\n" "mov z22.s, #0x0\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" "mov z21.s, #0x0\n" - "cmp %x[height], x15\n" - "ptrue p3.b\n" "mov z20.s, #0x0\n" - "csel x15, %x[height], x15, LT\n" "whilelt p12.b, XZR, %x[height]\n" - "whilelt p10.b, x14, %x[height]\n" - "whilelt p9.b, x11, %x[height]\n" - "whilelt p8.b, x10, %x[height]\n" + "whilelt p10.b, x15, %x[height]\n" + "whilelt p9.b, x14, %x[height]\n" + "whilelt p8.b, x13, %x[height]\n" "zip1 p12.b, p12.b, p9.b\n" "zip1 p10.b, p10.b, p8.b\n" - "cntb x9\n" + "ptrue p2.b\n" + "cntb x11\n" "ptrue p11.s\n" - "sub x15, x15, #0x1\n" + "sub x16, x16, #0x1\n" "zip1 p10.b, p12.b, p10.b\n" + "mov x10, %x[row_offset]\n" + "mov x9, %x[out]\n" "cbnz %x[first], 1f\n" - "addvl x16, x16, #-4\n" - "ld1w { z23.s }, p3/Z, [x16]\n" - "ld1w { z22.s }, p3/Z, [x16, #1, MUL VL]\n" - "ld1w { z21.s }, p3/Z, [x16, #2, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x16, #3, MUL VL]\n" + "addvl x9, x9, #-4\n" + "ld1w { z23.s }, p2/Z, [x9]\n" + "ld1w { z22.s }, p2/Z, [x9, #1, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x9, #3, MUL VL]\n" "1:" // Initialise row sums: End "mov x28, #0x0\n" "whilelt p9.b, x28, %x[width]\n" "whilelt p8.b, x28, %x[width]\n" "2:" // Width loop "add x27, %x[in], XZR, LSL #3\n" - "add x26, %x[in], x14, LSL #3\n" - "add x25, %x[in], x11, LSL #3\n" - "add x20, %x[in], x10, LSL #3\n" - "ldr x24, [x27], #0x8\n" - "mov x13, #0x0\n" - "ldr x23, [x26], #0x8\n" - "ldr x22, [x25], #0x8\n" - "ldr x21, [x20], #0x8\n" - "cbz x15, 4f\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x23, %x[in], x13, LSL #3\n" + "ldr x20, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x23], #0x8\n" + "cbz x16, 4f\n" "3:" // Loads: Loop - ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xe0112300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x17]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - "ldr x24, [x27], #0x8\n" - ".inst 0xe0112ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x17]\n" - "ldr x23, [x26], #0x8\n" - ".inst 0xe01126c2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x22, x17]\n" - "ldr x22, [x25], #0x8\n" - ".inst 0xe01122a3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x21, x17]\n" - "add x13, x13, #0x4\n" - "ldr x21, [x20], #0x8\n" - "cmp x13, x15, LSL #2\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe00a0320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x10]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe00a0281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x10]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0xe00a06c2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x22, x10]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe00a02a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x10]\n" + "add x12, x12, #0x4\n" + "cmp x12, x16, LSL #2\n" + "ldr x21, [x23], #0x8\n" "blt 3b\n" "4:" // Loads: Tail - ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" - ".inst 0x252d6142 // psel p2.b, p8.b/Z, p10.b[w13, #1]\n" - ".inst 0x25356141 // psel p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe00a0320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x10]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe00a0281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x10]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" "sub x20, %x[width], x28\n" - "cmp x20, x9\n" - "mov x12, #0x0\n" - ".inst 0xe0112300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x17]\n" - ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - "csel x20, x20, x9, LT\n" - ".inst 0xe0112ae1 // ld1b { za0h.b[x13, #1] }, p2/Z, [x23, x17]\n" + ".inst 0xe00a02c2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x22, x10]\n" + "cmp x20, x11\n" + "csel x20, x20, x11, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" "add x20, x20, #0x3\n" - ".inst 0xe01126c2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x22, x17]\n" + ".inst 0xe00a02a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x10]\n" + "mov x12, #0x0\n" "lsr x20, x20, #0x2\n" - ".inst 0xe01122a3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x21, x17]\n" "5:" // Stores: Loop ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0828c13 // mova z19.s, p3/M, za0v.s[x12]\n" + ".inst 0xe0bf8120 // st1w { za0v.s[x12] }, p0/Z, [x9, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + ".inst 0xe0af8124 // st1w { za1v.s[x12] }, p0/Z, [x9, x15, LSL #2]\n" ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0828c92 // mova z18.s, p3/M, za1v.s[x12]\n" - "udot z23.s, z19.b, z24.b\n" - ".inst 0xc0828d11 // mova z17.s, p3/M, za2v.s[x12]\n" - "udot z22.s, z18.b, z24.b\n" - ".inst 0xe0bf8200 // st1w { za0v.s[x12] }, p0/Z, [x16, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0828d90 // mova z16.s, p3/M, za3v.s[x12]\n" - "udot z21.s, z17.b, z24.b\n" - ".inst 0xe0ae8a04 // st1w { za1v.s[x12] }, p2/Z, [x16, x14, LSL #2]\n" - "udot z20.s, z16.b, z24.b\n" - ".inst 0xe0ab8608 // st1w { za2v.s[x12] }, p1/Z, [x16, x11, LSL #2]\n" - ".inst 0xe0aa820c // st1w { za3v.s[x12] }, p0/Z, [x16, x10, LSL #2]\n" + ".inst 0xc0828891 // mova z17.s, p2/M, za1v.s[x12]\n" + ".inst 0xe0ae8528 // st1w { za2v.s[x12] }, p1/Z, [x9, x14, LSL #2]\n" + ".inst 0xc0828913 // mova z19.s, p2/M, za2v.s[x12]\n" + "udot z23.s, z16.b, z24.b\n" + ".inst 0xe0ad812c // st1w { za3v.s[x12] }, p0/Z, [x9, x13, LSL #2]\n" + ".inst 0xc0828992 // mova z18.s, p2/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" - "addvl x16, x16, #4\n" "cmp x12, x20\n" + "udot z22.s, z17.b, z24.b\n" + "udot z21.s, z19.b, z24.b\n" + "addvl x9, x9, #4\n" + "udot z20.s, z18.b, z24.b\n" "blt 5b\n" "incb x28\n" - "incb x17\n" "whilelt p9.b, x28, %x[width]\n" "whilelt p8.b, x28, %x[width]\n" + "incb x10\n" "b.any 2b\n" - "st1w { z23.s }, p3, [x16]\n" - "st1w { z22.s }, p3, [x16, #1, MUL VL]\n" - "st1w { z21.s }, p3, [x16, #2, MUL VL]\n" - "st1w { z20.s }, p3, [x16, #3, MUL VL]\n" - "addvl x16, x16, #4\n" + "st1w { z23.s }, p2, [x9]\n" + "st1w { z22.s }, p2, [x9, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x9, #2, MUL VL]\n" + "st1w { z20.s }, p2, [x9, #3, MUL VL]\n" + "addvl x9, x9, #4\n" + "mov %x[out], x9\n" ".inst 0xd503467f // SMSTOP\n" - "mov %x[out], x16\n" : [out] "+&r" (out) : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp index 09c961f1cd..f6326100b7 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp @@ -32,92 +32,92 @@ void interleave_block<4, 1, VLType::SME, false>( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x16, #0x0\n" - "mov x15, %x[row_offset]\n" + "cntw x15\n" + "cmp %x[height], x15\n" "cntw x14\n" - "cntw x11\n" - "cmp %x[height], x14\n" - "cntw x10, ALL, MUL #2\n" - "cntw x9, ALL, MUL #3\n" - "csel x14, %x[height], x14, LT\n" + "cntw x13, ALL, MUL #2\n" + "cntw x11, ALL, MUL #3\n" + "csel x15, %x[height], x15, LT\n" + "mov x10, #0x0\n" "ptrue p4.s\n" - "sub x14, x14, #0x1\n" + "sub x15, x15, #0x1\n" "whilelt p3.s, XZR, %x[height]\n" - "whilelt p15.s, x11, %x[height]\n" - "whilelt p14.s, x10, %x[height]\n" - "whilelt p13.s, x9, %x[height]\n" + "whilelt p15.s, x14, %x[height]\n" + "whilelt p14.s, x13, %x[height]\n" + "whilelt p13.s, x11, %x[height]\n" + "mov x9, %x[row_offset]\n" "mov x28, %x[out]\n" - "whilelt p12.s, x16, %x[width]\n" - "whilelt p11.s, x16, %x[width]\n" - "whilelt p10.s, x16, %x[width]\n" - "whilelt p9.s, x16, %x[width]\n" - "whilelt p8.s, x16, %x[width]\n" + "whilelt p12.s, x10, %x[width]\n" + "whilelt p11.s, x10, %x[width]\n" + "whilelt p10.s, x10, %x[width]\n" + "whilelt p9.s, x10, %x[width]\n" + "whilelt p8.s, x10, %x[width]\n" "1:" // Width loop "add x27, %x[in], XZR, LSL #3\n" - "add x26, %x[in], x11, LSL #3\n" - "add x25, %x[in], x10, LSL #3\n" - "add x20, %x[in], x9, LSL #3\n" - "ldr x24, [x27], #0x8\n" - "mov x13, #0x0\n" + "add x26, %x[in], x14, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x13, LSL #3\n" + "add x20, %x[in], x11, LSL #3\n" "ldr x23, [x26], #0x8\n" - "ldr x22, [x25], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" "ldr x21, [x20], #0x8\n" - "cbz x14, 3f\n" + "cbz x15, 3f\n" "2:" // Loads: Loop - ".inst 0x25316c60 // psel p0.s, p11.s/Z, p3.s[w13]\n" - ".inst 0x253169e2 // psel p2.s, p10.s/Z, p15.s[w13]\n" - ".inst 0x253165c1 // psel p1.s, p9.s/Z, p14.s[w13]\n" - ".inst 0xe08f2300 // ld1w { za0h.s[x13] }, p0/Z, [x24, x15, LSL #2]\n" - ".inst 0x253161a0 // psel p0.s, p8.s/Z, p13.s[w13]\n" - "ldr x24, [x27], #0x8\n" - ".inst 0xe08f2ae4 // ld1w { za1h.s[x13] }, p2/Z, [x23, x15, LSL #2]\n" + ".inst 0x25306c60 // psel p0.s, p11.s/Z, p3.s[w12]\n" + ".inst 0x253069e2 // psel p2.s, p10.s/Z, p15.s[w12]\n" + ".inst 0xe0890320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x9, LSL #2]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0x253065c1 // psel p1.s, p9.s/Z, p14.s[w12]\n" + ".inst 0x253061a0 // psel p0.s, p8.s/Z, p13.s[w12]\n" + ".inst 0xe0890ae4 // ld1w { za1h.s[x12] }, p2/Z, [x23, x9, LSL #2]\n" "ldr x23, [x26], #0x8\n" - ".inst 0xe08f26c8 // ld1w { za2h.s[x13] }, p1/Z, [x22, x15, LSL #2]\n" - "ldr x22, [x25], #0x8\n" - ".inst 0xe08f22ac // ld1w { za3h.s[x13] }, p0/Z, [x21, x15, LSL #2]\n" - "add x13, x13, #0x1\n" + ".inst 0xe08906c8 // ld1w { za2h.s[x12] }, p1/Z, [x22, x9, LSL #2]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe08902ac // ld1w { za3h.s[x12] }, p0/Z, [x21, x9, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x15\n" "ldr x21, [x20], #0x8\n" - "cmp x13, x14\n" "blt 2b\n" "3:" // Loads: Tail - ".inst 0x25316c60 // psel p0.s, p11.s/Z, p3.s[w13]\n" - ".inst 0x253169e2 // psel p2.s, p10.s/Z, p15.s[w13]\n" - ".inst 0x253165c1 // psel p1.s, p9.s/Z, p14.s[w13]\n" - "sub x20, %x[width], x16\n" - "cmp x20, x11\n" + "sub x20, %x[width], x10\n" + ".inst 0x25306c60 // psel p0.s, p11.s/Z, p3.s[w12]\n" + ".inst 0xe0890320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x9, LSL #2]\n" + ".inst 0x253069e0 // psel p0.s, p10.s/Z, p15.s[w12]\n" + ".inst 0x253065c1 // psel p1.s, p9.s/Z, p14.s[w12]\n" + ".inst 0xe08902e4 // ld1w { za1h.s[x12] }, p0/Z, [x23, x9, LSL #2]\n" + ".inst 0x253061a0 // psel p0.s, p8.s/Z, p13.s[w12]\n" + "cmp x20, x14\n" + ".inst 0xe08906c8 // ld1w { za2h.s[x12] }, p1/Z, [x22, x9, LSL #2]\n" + ".inst 0xe08902ac // ld1w { za3h.s[x12] }, p0/Z, [x21, x9, LSL #2]\n" "mov x12, #0x0\n" - ".inst 0xe08f2300 // ld1w { za0h.s[x13] }, p0/Z, [x24, x15, LSL #2]\n" - ".inst 0x253161a0 // psel p0.s, p8.s/Z, p13.s[w13]\n" - "csel x20, x20, x11, LT\n" - ".inst 0xe08f2ae4 // ld1w { za1h.s[x13] }, p2/Z, [x23, x15, LSL #2]\n" - ".inst 0xe08f26c8 // ld1w { za2h.s[x13] }, p1/Z, [x22, x15, LSL #2]\n" - ".inst 0xe08f22ac // ld1w { za3h.s[x13] }, p0/Z, [x21, x15, LSL #2]\n" + "csel x20, x20, x14, LT\n" "4:" // Stores: Loop ".inst 0x25305180 // psel p0.s, p4.s/Z, p12.s[w12]\n" - ".inst 0x25305182 // psel p2.s, p4.s/Z, p12.s[w12]\n" - ".inst 0x25305181 // psel p1.s, p4.s/Z, p12.s[w12]\n" ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" ".inst 0x25305180 // psel p0.s, p4.s/Z, p12.s[w12]\n" - ".inst 0xe0ab8b84 // st1w { za1v.s[x12] }, p2/Z, [x28, x11, LSL #2]\n" - ".inst 0xe0aa8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x10, LSL #2]\n" - ".inst 0xe0a9838c // st1w { za3v.s[x12] }, p0/Z, [x28, x9, LSL #2]\n" + ".inst 0xe0ae8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x14, LSL #2]\n" + ".inst 0x25305181 // psel p1.s, p4.s/Z, p12.s[w12]\n" + ".inst 0x25305180 // psel p0.s, p4.s/Z, p12.s[w12]\n" + ".inst 0xe0ad8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x13, LSL #2]\n" + ".inst 0xe0ab838c // st1w { za3v.s[x12] }, p0/Z, [x28, x11, LSL #2]\n" "add x12, x12, #0x1\n" - "addvl x28, x28, #4\n" "cmp x12, x20\n" + "addvl x28, x28, #4\n" "blt 4b\n" - "incw x16\n" - "incw x15\n" - "whilelt p12.s, x16, %x[width]\n" - "whilelt p11.s, x16, %x[width]\n" - "whilelt p10.s, x16, %x[width]\n" - "whilelt p9.s, x16, %x[width]\n" - "whilelt p8.s, x16, %x[width]\n" + "incw x10\n" + "whilelt p12.s, x10, %x[width]\n" + "whilelt p11.s, x10, %x[width]\n" + "whilelt p10.s, x10, %x[width]\n" + "whilelt p9.s, x10, %x[width]\n" + "whilelt p8.s, x10, %x[width]\n" + "incw x9\n" "b.any 1b\n" "mov %x[out], x28\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } |