diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/indirect-interleaves')
55 files changed, 12638 insertions, 0 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp new file mode 100644 index 0000000000..807511f0d2 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2017-2018 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __arm__ + +#include <arm_neon.h> + +#include "../asmlib.hpp" + +template<> +void interleave_block<6, 1, VLType::None, false>( + float * &outptr, const float * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + const float *inptr0 = in[0] + row_offset; + const float *inptr1 = in[1] + row_offset; + const float *inptr2 = in[2] + row_offset; + const float *inptr3 = in[3] + row_offset; + const float *inptr4 = in[4] + row_offset; + const float *inptr5 = in[5] + row_offset; + + // Cope with ragged cases by aliasing the first row (which is always valid). + // The nonsense output produced will be suppressed later anyway. + switch (height) { + case 1: + inptr1 = inptr0; + // fall through + case 2: + inptr2 = inptr0; + // fall through + case 3: + inptr3 = inptr0; + // fall through + case 4: + inptr4 = inptr0; + // fall through + case 5: + inptr5 = inptr0; + // fall through + default: + case 6: + break; + } + + //prefetch_2x(inptr0); + //prefetch_2x(inptr1); + //prefetch_2x(inptr2); + //prefetch_2x(inptr3); + //prefetch_2x(inptr4); + //prefetch_2x(inptr5); + + for (;width>7;width-=8) { + __asm __volatile ( + // Load up 8 elements (2 vectors) from each of 8 sources. + "VLD1.32 {d0-d3}, [%[inptr0]]!\n" // q0=A0A1A2A3 + "VLD1.32 {d4-d7}, [%[inptr1]]!\n" // q2=B0B1B2B3 + "VLD1.32 {d8-d11}, [%[inptr2]]!\n" // q4=C0C1C2C3 + "VZIP.32 q0, q4\n" // q0=A0C0A1C1, q4 = A2C2A3C3 + "VLD1.32 {d12-d15}, [%[inptr3]]!\n" // q6=D0D1D2D3 + "VZIP.32 q2, q6\n" // q2=B0D0B1D1, q6 = B2D2B3D3 + "VLD1.32 {d16-d19}, [%[inptr4]]!\n" + "VLD1.32 {d20-d23}, [%[inptr5]]!\n" + "VZIP.32 q8, q10\n" // q8=E0F0E1F1, q10 = E2F2E3F3 + ASM_PREFETCH("[%[inptr0], #128]") + "VZIP.32 q0, q2\n" // q0 = A0B0C0D0, q2 = A1B1C1D1 + + // Store first elements + "VST1.32 {d0-d1}, [%[outptr]]!\n" + "VST1.32 {d16}, [%[outptr]]!\n" + + "VZIP.32 q4, q6\n" // q4 = A2B2C2D2, q6 = A3B3C3D3 + + // Store second elements + "VST1.32 {d4-d5}, [%[outptr]]!\n" + "VZIP.32 q1, q5\n" + ASM_PREFETCH("[%[inptr1], #128]") + "VST1.32 {d17}, [%[outptr]]!\n" + "VZIP.32 q3, q7\n" + + // Store third elements + "VZIP.32 q9, q11\n" + "VST1.32 {d8-d9}, [%[outptr]]!\n" + "VZIP.32 q1, q3\n" + ASM_PREFETCH("[%[inptr2], #128]") + "VST1.32 {d20}, [%[outptr]]!\n" + + // Store fourth elements + "VZIP.32 q5, q7\n" + "VST1.32 {d12-d13}, [%[outptr]]!\n" + ASM_PREFETCH("[%[inptr3], #128]") + "VST1.32 {d21}, [%[outptr]]!\n" + + // Fifth + "VST1.32 {d2-d3}, [%[outptr]]!\n" + ASM_PREFETCH("[%[inptr4], #128]") + "VST1.32 {d18}, [%[outptr]]!\n" + + // Sixth + "VST1.32 {d6-d7}, [%[outptr]]!\n" + ASM_PREFETCH("[%[inptr5], #128]") + "VST1.32 {d19}, [%[outptr]]!\n" + + // Seventh + "VST1.32 {d10-d11}, [%[outptr]]!\n" + "VST1.32 {d22}, [%[outptr]]!\n" + + // Eighth + "VST1.32 {d14-d15}, [%[outptr]]!\n" + "VST1.32 {d23}, [%[outptr]]!\n" + + : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), + [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [outptr] "+r" (outptr) + : + : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "memory" + ); + } + + for (;width>0;width--) { + *outptr++ = *inptr0++; + *outptr++ = *inptr1++; + *outptr++ = *inptr2++; + *outptr++ = *inptr3++; + *outptr++ = *inptr4++; + *outptr++ = *inptr5++; + } +} + +#endif // __arm__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp new file mode 100644 index 0000000000..e4bfc0f6e4 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<4, 16, VLType::None, false>( + int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x23, [%x[in], #0x0]\n" + "ldr x22, [%x[in], #0x8]\n" + "cmp %x[height], #0x4\n" + "add x23, x23, %x[row_offset]\n" + "ldr x21, [%x[in], #0x10]\n" + "ldr x20, [%x[in], #0x18]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "add x20, x20, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "mov x20, x23\n" + "csel x22, x22, x23, GE\n" + "csel x21, x21, x23, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x10\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x20, #0x0]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "prfm pldl1keep, [x20, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q19, [x23], #0x10\n" + "ldr q18, [x22], #0x10\n" + "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" + "ldr q17, [x21], #0x10\n" + "ldr q16, [x20], #0x10\n" + "str q19, [%x[out_ptr], #0x0]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "prfm pldl1keep, [x22, #0x70]\n" + "str q18, [%x[out_ptr], #0x10]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "prfm pldl1keep, [x20, #0x70]\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 12f\n" + "tbz %x[width], #3, 7f\n" + "ldr d19, [x23], #0x8\n" + "ldr d18, [x22], #0x8\n" + "ldr d17, [x21], #0x8\n" + "ldr d16, [x20], #0x8\n" + "tbz %x[width], #2, 5f\n" + "ld1 { v19.s }[2], [x23], #0x4\n" + "ld1 { v18.s }[2], [x22], #0x4\n" + "ld1 { v17.s }[2], [x21], #0x4\n" + "ld1 { v16.s }[2], [x20], #0x4\n" + "tbz %x[width], #1, 4f\n" + "ld1 { v19.h }[6], [x23], #0x2\n" + "ld1 { v18.h }[6], [x22], #0x2\n" + "ld1 { v17.h }[6], [x21], #0x2\n" + "ld1 { v16.h }[6], [x20], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v19.b }[14], [x23]\n" + "ld1 { v18.b }[14], [x22]\n" + "ld1 { v17.b }[14], [x21]\n" + "ld1 { v16.b }[14], [x20]\n" + "b 11f\n" + "4:" // odd_loads_1_12 + "tbz %x[width], #0, 11f\n" + "ld1 { v19.b }[12], [x23]\n" + "ld1 { v18.b }[12], [x22]\n" + "ld1 { v17.b }[12], [x21]\n" + "ld1 { v16.b }[12], [x20]\n" + "b 11f\n" + "5:" // odd_loads_2_8 + "tbz %x[width], #1, 6f\n" + "ld1 { v19.h }[4], [x23], #0x2\n" + "ld1 { v18.h }[4], [x22], #0x2\n" + "ld1 { v17.h }[4], [x21], #0x2\n" + "ld1 { v16.h }[4], [x20], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v19.b }[10], [x23]\n" + "ld1 { v18.b }[10], [x22]\n" + "ld1 { v17.b }[10], [x21]\n" + "ld1 { v16.b }[10], [x20]\n" + "b 11f\n" + "6:" // odd_loads_1_8 + "tbz %x[width], #0, 11f\n" + "ld1 { v19.b }[8], [x23]\n" + "ld1 { v18.b }[8], [x22]\n" + "ld1 { v17.b }[8], [x21]\n" + "ld1 { v16.b }[8], [x20]\n" + "b 11f\n" + "7:" // odd_loads_4_0 + "tbz %x[width], #2, 9f\n" + "ldr s19, [x23], #0x4\n" + "ldr s18, [x22], #0x4\n" + "ldr s17, [x21], #0x4\n" + "ldr s16, [x20], #0x4\n" + "tbz %x[width], #1, 8f\n" + "ld1 { v19.h }[2], [x23], #0x2\n" + "ld1 { v18.h }[2], [x22], #0x2\n" + "ld1 { v17.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v19.b }[6], [x23]\n" + "ld1 { v18.b }[6], [x22]\n" + "ld1 { v17.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" + "b 11f\n" + "8:" // odd_loads_1_4 + "tbz %x[width], #0, 11f\n" + "ld1 { v19.b }[4], [x23]\n" + "ld1 { v18.b }[4], [x22]\n" + "ld1 { v17.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" + "b 11f\n" + "9:" // odd_loads_2_0 + "tbz %x[width], #1, 10f\n" + "ldr h19, [x23], #0x2\n" + "ldr h18, [x22], #0x2\n" + "ldr h17, [x21], #0x2\n" + "ldr h16, [x20], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v19.b }[2], [x23]\n" + "ld1 { v18.b }[2], [x22]\n" + "ld1 { v17.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" + "b 11f\n" + "10:" // odd_loads_1_0 + "ldr b19, [x23, #0x0]\n" + "ldr b18, [x22, #0x0]\n" + "ldr b17, [x21, #0x0]\n" + "ldr b16, [x20, #0x0]\n" + "11:" // Odd load end + "str q19, [%x[out_ptr], #0x0]\n" + "str q18, [%x[out_ptr], #0x10]\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "12:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "x20", "x21", "x22", "x23" + ); +} + +template<> +void interleave_block<4, 16, VLType::None, false>( + uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + int8_t * &out_cast = reinterpret_cast<int8_t * &>(out_ptr); + const int8_t * const * in_cast = reinterpret_cast<const int8_t * const *>(in); + + interleave_block<4, 16, VLType::None, false>(out_cast, in_cast, width, height, row_offset, false); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp new file mode 100644 index 0000000000..23800edf20 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<4, 16, VLType::None, true>( + int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x24, [%x[in], #0x0]\n" + "ldr x23, [%x[in], #0x8]\n" + "cmp %x[height], #0x4\n" + "mov x22, #0x0\n" + "ldr x21, [%x[in], #0x10]\n" + "ldr x20, [%x[in], #0x18]\n" + "movi v28.8h, #0x0\n" + "movi v27.8h, #0x0\n" + "movi v26.8h, #0x0\n" + "movi v25.8h, #0x0\n" + "add x24, x24, %x[row_offset]\n" + "add x23, x23, %x[row_offset]\n" + "movi v24.4s, #0x0\n" + "movi v23.4s, #0x0\n" + "add x21, x21, %x[row_offset]\n" + "add x20, x20, %x[row_offset]\n" + "movi v22.4s, #0x0\n" + "movi v21.4s, #0x0\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "mov x20, x24\n" + "csel x23, x23, x24, GE\n" + "csel x21, x21, x24, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "movi v20.4s, #0x0\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x20, #0x0]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "prfm pldl1keep, [x20, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x10\n" + "ld1 { v20.4s }, [%x[out_ptr]]\n" + "2:" // first_pass + "cmp %x[width], #0x10\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x22, #0x7e\n" + "ble 4f\n" + "sadalp v24.4s, v28.8h\n" + "movi v28.8h, #0x0\n" + "mov x22, #0x0\n" + "sadalp v23.4s, v27.8h\n" + "movi v27.8h, #0x0\n" + "sadalp v22.4s, v26.8h\n" + "movi v26.8h, #0x0\n" + "sadalp v21.4s, v25.8h\n" + "movi v25.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr q19, [x24], #0x10\n" + "ldr q18, [x23], #0x10\n" + "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" + "ldr q17, [x21], #0x10\n" + "ldr q16, [x20], #0x10\n" + "str q19, [%x[out_ptr], #0x0]\n" + "sadalp v28.8h, v19.16b\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "str q18, [%x[out_ptr], #0x10]\n" + "sadalp v27.8h, v18.16b\n" + "prfm pldl1keep, [x21, #0x70]\n" + "prfm pldl1keep, [x20, #0x70]\n" + "str q17, [%x[out_ptr], #0x20]\n" + "sadalp v26.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x30]\n" + "sadalp v25.8h, v16.16b\n" + "add x22, x22, #0x1\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 14f\n" + "tbz %x[width], #3, 9f\n" + "ldr d19, [x24], #0x8\n" + "ldr d18, [x23], #0x8\n" + "ldr d17, [x21], #0x8\n" + "ldr d16, [x20], #0x8\n" + "tbz %x[width], #2, 7f\n" + "ld1 { v19.s }[2], [x24], #0x4\n" + "ld1 { v18.s }[2], [x23], #0x4\n" + "ld1 { v17.s }[2], [x21], #0x4\n" + "ld1 { v16.s }[2], [x20], #0x4\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v19.h }[6], [x24], #0x2\n" + "ld1 { v18.h }[6], [x23], #0x2\n" + "ld1 { v17.h }[6], [x21], #0x2\n" + "ld1 { v16.h }[6], [x20], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[14], [x24]\n" + "ld1 { v18.b }[14], [x23]\n" + "ld1 { v17.b }[14], [x21]\n" + "ld1 { v16.b }[14], [x20]\n" + "b 13f\n" + "6:" // odd_loads_1_12 + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[12], [x24]\n" + "ld1 { v18.b }[12], [x23]\n" + "ld1 { v17.b }[12], [x21]\n" + "ld1 { v16.b }[12], [x20]\n" + "b 13f\n" + "7:" // odd_loads_2_8 + "tbz %x[width], #1, 8f\n" + "ld1 { v19.h }[4], [x24], #0x2\n" + "ld1 { v18.h }[4], [x23], #0x2\n" + "ld1 { v17.h }[4], [x21], #0x2\n" + "ld1 { v16.h }[4], [x20], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[10], [x24]\n" + "ld1 { v18.b }[10], [x23]\n" + "ld1 { v17.b }[10], [x21]\n" + "ld1 { v16.b }[10], [x20]\n" + "b 13f\n" + "8:" // odd_loads_1_8 + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[8], [x24]\n" + "ld1 { v18.b }[8], [x23]\n" + "ld1 { v17.b }[8], [x21]\n" + "ld1 { v16.b }[8], [x20]\n" + "b 13f\n" + "9:" // odd_loads_4_0 + "tbz %x[width], #2, 11f\n" + "ldr s19, [x24], #0x4\n" + "ldr s18, [x23], #0x4\n" + "ldr s17, [x21], #0x4\n" + "ldr s16, [x20], #0x4\n" + "tbz %x[width], #1, 10f\n" + "ld1 { v19.h }[2], [x24], #0x2\n" + "ld1 { v18.h }[2], [x23], #0x2\n" + "ld1 { v17.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[6], [x24]\n" + "ld1 { v18.b }[6], [x23]\n" + "ld1 { v17.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" + "b 13f\n" + "10:" // odd_loads_1_4 + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[4], [x24]\n" + "ld1 { v18.b }[4], [x23]\n" + "ld1 { v17.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" + "b 13f\n" + "11:" // odd_loads_2_0 + "tbz %x[width], #1, 12f\n" + "ldr h19, [x24], #0x2\n" + "ldr h18, [x23], #0x2\n" + "ldr h17, [x21], #0x2\n" + "ldr h16, [x20], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[2], [x24]\n" + "ld1 { v18.b }[2], [x23]\n" + "ld1 { v17.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" + "b 13f\n" + "12:" // odd_loads_1_0 + "ldr b19, [x24, #0x0]\n" + "ldr b18, [x23, #0x0]\n" + "ldr b17, [x21, #0x0]\n" + "ldr b16, [x20, #0x0]\n" + "13:" // Odd load end + "str q19, [%x[out_ptr], #0x0]\n" + "sadalp v28.8h, v19.16b\n" + "sadalp v27.8h, v18.16b\n" + "str q18, [%x[out_ptr], #0x10]\n" + "sadalp v26.8h, v17.16b\n" + "sadalp v25.8h, v16.16b\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "14:" // Odds skip + "sadalp v24.4s, v28.8h\n" + "sadalp v23.4s, v27.8h\n" + "sadalp v22.4s, v26.8h\n" + "sadalp v21.4s, v25.8h\n" + "addp v24.4s, v24.4s, v23.4s\n" + "addp v16.4s, v22.4s, v21.4s\n" + "addp v24.4s, v24.4s, v16.4s\n" + "add v24.4s, v24.4s, v20.4s\n" + "str q24, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp new file mode 100644 index 0000000000..15545c24db --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<4, 16, VLType::None, true>( + uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x24, [%x[in], #0x0]\n" + "ldr x23, [%x[in], #0x8]\n" + "cmp %x[height], #0x4\n" + "mov x22, #0x0\n" + "ldr x21, [%x[in], #0x10]\n" + "ldr x20, [%x[in], #0x18]\n" + "movi v28.8h, #0x0\n" + "movi v27.8h, #0x0\n" + "movi v26.8h, #0x0\n" + "movi v25.8h, #0x0\n" + "add x24, x24, %x[row_offset]\n" + "add x23, x23, %x[row_offset]\n" + "movi v24.4s, #0x0\n" + "movi v23.4s, #0x0\n" + "add x21, x21, %x[row_offset]\n" + "add x20, x20, %x[row_offset]\n" + "movi v22.4s, #0x0\n" + "movi v21.4s, #0x0\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "mov x20, x24\n" + "csel x23, x23, x24, GE\n" + "csel x21, x21, x24, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "movi v20.4s, #0x0\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x20, #0x0]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "prfm pldl1keep, [x20, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x10\n" + "ld1 { v20.4s }, [%x[out_ptr]]\n" + "2:" // first_pass + "cmp %x[width], #0x10\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x22, #0x7e\n" + "ble 4f\n" + "uadalp v24.4s, v28.8h\n" + "movi v28.8h, #0x0\n" + "mov x22, #0x0\n" + "uadalp v23.4s, v27.8h\n" + "movi v27.8h, #0x0\n" + "uadalp v22.4s, v26.8h\n" + "movi v26.8h, #0x0\n" + "uadalp v21.4s, v25.8h\n" + "movi v25.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr q19, [x24], #0x10\n" + "ldr q18, [x23], #0x10\n" + "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" + "ldr q17, [x21], #0x10\n" + "ldr q16, [x20], #0x10\n" + "str q19, [%x[out_ptr], #0x0]\n" + "uadalp v28.8h, v19.16b\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "str q18, [%x[out_ptr], #0x10]\n" + "uadalp v27.8h, v18.16b\n" + "prfm pldl1keep, [x21, #0x70]\n" + "prfm pldl1keep, [x20, #0x70]\n" + "str q17, [%x[out_ptr], #0x20]\n" + "uadalp v26.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x30]\n" + "uadalp v25.8h, v16.16b\n" + "add x22, x22, #0x1\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 14f\n" + "tbz %x[width], #3, 9f\n" + "ldr d19, [x24], #0x8\n" + "ldr d18, [x23], #0x8\n" + "ldr d17, [x21], #0x8\n" + "ldr d16, [x20], #0x8\n" + "tbz %x[width], #2, 7f\n" + "ld1 { v19.s }[2], [x24], #0x4\n" + "ld1 { v18.s }[2], [x23], #0x4\n" + "ld1 { v17.s }[2], [x21], #0x4\n" + "ld1 { v16.s }[2], [x20], #0x4\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v19.h }[6], [x24], #0x2\n" + "ld1 { v18.h }[6], [x23], #0x2\n" + "ld1 { v17.h }[6], [x21], #0x2\n" + "ld1 { v16.h }[6], [x20], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[14], [x24]\n" + "ld1 { v18.b }[14], [x23]\n" + "ld1 { v17.b }[14], [x21]\n" + "ld1 { v16.b }[14], [x20]\n" + "b 13f\n" + "6:" // odd_loads_1_12 + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[12], [x24]\n" + "ld1 { v18.b }[12], [x23]\n" + "ld1 { v17.b }[12], [x21]\n" + "ld1 { v16.b }[12], [x20]\n" + "b 13f\n" + "7:" // odd_loads_2_8 + "tbz %x[width], #1, 8f\n" + "ld1 { v19.h }[4], [x24], #0x2\n" + "ld1 { v18.h }[4], [x23], #0x2\n" + "ld1 { v17.h }[4], [x21], #0x2\n" + "ld1 { v16.h }[4], [x20], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[10], [x24]\n" + "ld1 { v18.b }[10], [x23]\n" + "ld1 { v17.b }[10], [x21]\n" + "ld1 { v16.b }[10], [x20]\n" + "b 13f\n" + "8:" // odd_loads_1_8 + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[8], [x24]\n" + "ld1 { v18.b }[8], [x23]\n" + "ld1 { v17.b }[8], [x21]\n" + "ld1 { v16.b }[8], [x20]\n" + "b 13f\n" + "9:" // odd_loads_4_0 + "tbz %x[width], #2, 11f\n" + "ldr s19, [x24], #0x4\n" + "ldr s18, [x23], #0x4\n" + "ldr s17, [x21], #0x4\n" + "ldr s16, [x20], #0x4\n" + "tbz %x[width], #1, 10f\n" + "ld1 { v19.h }[2], [x24], #0x2\n" + "ld1 { v18.h }[2], [x23], #0x2\n" + "ld1 { v17.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[6], [x24]\n" + "ld1 { v18.b }[6], [x23]\n" + "ld1 { v17.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" + "b 13f\n" + "10:" // odd_loads_1_4 + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[4], [x24]\n" + "ld1 { v18.b }[4], [x23]\n" + "ld1 { v17.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" + "b 13f\n" + "11:" // odd_loads_2_0 + "tbz %x[width], #1, 12f\n" + "ldr h19, [x24], #0x2\n" + "ldr h18, [x23], #0x2\n" + "ldr h17, [x21], #0x2\n" + "ldr h16, [x20], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v19.b }[2], [x24]\n" + "ld1 { v18.b }[2], [x23]\n" + "ld1 { v17.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" + "b 13f\n" + "12:" // odd_loads_1_0 + "ldr b19, [x24, #0x0]\n" + "ldr b18, [x23, #0x0]\n" + "ldr b17, [x21, #0x0]\n" + "ldr b16, [x20, #0x0]\n" + "13:" // Odd load end + "str q19, [%x[out_ptr], #0x0]\n" + "uadalp v28.8h, v19.16b\n" + "uadalp v27.8h, v18.16b\n" + "str q18, [%x[out_ptr], #0x10]\n" + "uadalp v26.8h, v17.16b\n" + "uadalp v25.8h, v16.16b\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "14:" // Odds skip + "uadalp v24.4s, v28.8h\n" + "uadalp v23.4s, v27.8h\n" + "uadalp v22.4s, v26.8h\n" + "uadalp v21.4s, v25.8h\n" + "addp v24.4s, v24.4s, v23.4s\n" + "addp v16.4s, v22.4s, v21.4s\n" + "addp v24.4s, v24.4s, v16.4s\n" + "add v24.4s, v24.4s, v20.4s\n" + "str q24, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp new file mode 100644 index 0000000000..b900c330b7 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, false>( + float * &out_ptr, const bfloat16 * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "movi v16.8h, #0x0\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x28, x28, %x[row_offset], LSL #1\n" + "add x27, x27, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x26, x26, %x[row_offset], LSL #1\n" + "add x25, x25, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x24, x24, %x[row_offset], LSL #1\n" + "add x23, x23, %x[row_offset], LSL #1\n" + "add x22, x22, %x[row_offset], LSL #1\n" + "add x21, x21, %x[row_offset], LSL #1\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x4\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr d27, [x28], #0x8\n" + "ldr d26, [x27], #0x8\n" + "shll v27.4s, v27.4h, #0x10\n" + "shll v26.4s, v26.4h, #0x10\n" + "ldr d22, [x26], #0x8\n" + "ldr d21, [x25], #0x8\n" + "shll v22.4s, v22.4h, #0x10\n" + "shll v21.4s, v21.4h, #0x10\n" + "ldr d20, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "shll v20.4s, v20.4h, #0x10\n" + "shll v25.4s, v25.4h, #0x10\n" + "ldr d19, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" + "shll v19.4s, v19.4h, #0x10\n" + "shll v16.4s, v16.4h, #0x10\n" + "zip1 v24.4s, v27.4s, v22.4s\n" + "zip1 v23.4s, v26.4s, v21.4s\n" + "subs %x[width], %x[width], #0x4\n" + "cmp %x[width], #0x4\n" + "zip1 v18.4s, v20.4s, v19.4s\n" + "zip1 v17.4s, v25.4s, v16.4s\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip2 v22.4s, v27.4s, v22.4s\n" + "zip2 v21.4s, v26.4s, v21.4s\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip2 v20.4s, v20.4s, v19.4s\n" + "zip2 v19.4s, v25.4s, v16.4s\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip1 v16.4s, v24.4s, v23.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v16.4s, v24.4s, v23.4s\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v17.4s, v18.4s, v17.4s\n" + "zip1 v16.4s, v22.4s, v21.4s\n" + "str q17, [%x[out_ptr], #0x30]\n" + "zip1 v18.4s, v20.4s, v19.4s\n" + "zip2 v17.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 6f\n" + "tbz %x[width], #1, 4f\n" + "ldr s28, [x28], #0x4\n" + "ldr s27, [x27], #0x4\n" + "mov x20, #0x2\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" + "ldr s24, [x24], #0x4\n" + "ldr s23, [x23], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" + "tbz %x[width], #0, 5f\n" + "ld1 { v28.h }[2], [x28]\n" + "ld1 { v27.h }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v26.h }[2], [x26]\n" + "ld1 { v25.h }[2], [x25]\n" + "ld1 { v24.h }[2], [x24]\n" + "ld1 { v23.h }[2], [x23]\n" + "ld1 { v22.h }[2], [x22]\n" + "ld1 { v21.h }[2], [x21]\n" + "b 5f\n" + "4:" // odd_loads_1_0 + "ldr h28, [x28, #0x0]\n" + "ldr h27, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr h26, [x26, #0x0]\n" + "ldr h25, [x25, #0x0]\n" + "ldr h24, [x24, #0x0]\n" + "ldr h23, [x23, #0x0]\n" + "ldr h22, [x22, #0x0]\n" + "ldr h21, [x21, #0x0]\n" + "5:" // Odd load end + "shll v28.4s, v28.4h, #0x10\n" + "shll v27.4s, v27.4h, #0x10\n" + "subs x20, x20, #0x1\n" + "shll v26.4s, v26.4h, #0x10\n" + "shll v25.4s, v25.4h, #0x10\n" + "shll v24.4s, v24.4h, #0x10\n" + "shll v23.4s, v23.4h, #0x10\n" + "shll v22.4s, v22.4h, #0x10\n" + "shll v21.4s, v21.4h, #0x10\n" + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 6f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 6f\n" + "zip2 v19.4s, v28.4s, v26.4s\n" + "zip2 v16.4s, v27.4s, v25.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v19.4s, v16.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "6:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp new file mode 100644 index 0000000000..e54b3b9f41 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, false>( + __fp16 * &out_ptr, const __fp16 * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset], LSL #1\n" + "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset], LSL #1\n" + "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset], LSL #1\n" + "add x22, x22, %x[row_offset], LSL #1\n" + "add x21, x21, %x[row_offset], LSL #1\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x8\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q25, [x28], #0x10\n" + "ldr q27, [x27], #0x10\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "ldr q26, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "ldr q21, [x24], #0x10\n" + "ldr q20, [x23], #0x10\n" + "zip1 v23.8h, v25.8h, v21.8h\n" + "zip1 v22.8h, v27.8h, v20.8h\n" + "ldr q17, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v19.8h, v26.8h, v17.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" + "zip2 v25.8h, v25.8h, v21.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip1 v24.8h, v23.8h, v19.8h\n" + "zip1 v17.8h, v22.8h, v18.8h\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip2 v23.8h, v23.8h, v19.8h\n" + "zip2 v19.8h, v22.8h, v18.8h\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip1 v22.8h, v25.8h, v21.8h\n" + "zip1 v18.8h, v20.8h, v16.8h\n" + "zip2 v21.8h, v25.8h, v21.8h\n" + "zip2 v20.8h, v20.8h, v16.8h\n" + "zip1 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v23.8h, v19.8h\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip1 v19.8h, v22.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v18.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "zip2 v16.8h, v21.8h, v20.8h\n" + "str q19, [%x[out_ptr], #0x40]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 8f\n" + "tbz %x[width], #2, 5f\n" + "ldr d30, [x28], #0x8\n" + "ldr d29, [x27], #0x8\n" + "ldr d28, [x26], #0x8\n" + "ldr d27, [x25], #0x8\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "ldr d24, [x22], #0x8\n" + "ldr d23, [x21], #0x8\n" + "tbz %x[width], #1, 4f\n" + "ld1 { v30.s }[2], [x28], #0x4\n" + "ld1 { v29.s }[2], [x27], #0x4\n" + "mov x20, #0x6\n" + "ld1 { v28.s }[2], [x26], #0x4\n" + "ld1 { v27.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x24], #0x4\n" + "ld1 { v25.s }[2], [x23], #0x4\n" + "ld1 { v24.s }[2], [x22], #0x4\n" + "ld1 { v23.s }[2], [x21], #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.h }[6], [x28]\n" + "ld1 { v29.h }[6], [x27]\n" + "mov x20, #0x7\n" + "ld1 { v28.h }[6], [x26]\n" + "ld1 { v27.h }[6], [x25]\n" + "ld1 { v26.h }[6], [x24]\n" + "ld1 { v25.h }[6], [x23]\n" + "ld1 { v24.h }[6], [x22]\n" + "ld1 { v23.h }[6], [x21]\n" + "b 7f\n" + "4:" // odd_loads_1_4 + "mov x20, #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.h }[4], [x28]\n" + "ld1 { v29.h }[4], [x27]\n" + "mov x20, #0x5\n" + "ld1 { v28.h }[4], [x26]\n" + "ld1 { v27.h }[4], [x25]\n" + "ld1 { v26.h }[4], [x24]\n" + "ld1 { v25.h }[4], [x23]\n" + "ld1 { v24.h }[4], [x22]\n" + "ld1 { v23.h }[4], [x21]\n" + "b 7f\n" + "5:" // odd_loads_2_0 + "tbz %x[width], #1, 6f\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "mov x20, #0x2\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.h }[2], [x28]\n" + "ld1 { v29.h }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v28.h }[2], [x26]\n" + "ld1 { v27.h }[2], [x25]\n" + "ld1 { v26.h }[2], [x24]\n" + "ld1 { v25.h }[2], [x23]\n" + "ld1 { v24.h }[2], [x22]\n" + "ld1 { v23.h }[2], [x21]\n" + "b 7f\n" + "6:" // odd_loads_1_0 + "ldr h30, [x28, #0x0]\n" + "ldr h29, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr h28, [x26, #0x0]\n" + "ldr h27, [x25, #0x0]\n" + "ldr h26, [x24, #0x0]\n" + "ldr h25, [x23, #0x0]\n" + "ldr h24, [x22, #0x0]\n" + "ldr h23, [x21, #0x0]\n" + "7:" // Odd load end + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "8:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp new file mode 100644 index 0000000000..3a5dcf4a6b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, false>( + float * &out_ptr, const __fp16 * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset], LSL #1\n" + "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset], LSL #1\n" + "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset], LSL #1\n" + "add x22, x22, %x[row_offset], LSL #1\n" + "add x21, x21, %x[row_offset], LSL #1\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x4\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr d27, [x28], #0x8\n" + "ldr d26, [x27], #0x8\n" + "fcvtl v27.4s, v27.4h\n" + "fcvtl v26.4s, v26.4h\n" + "ldr d22, [x26], #0x8\n" + "ldr d21, [x25], #0x8\n" + "fcvtl v22.4s, v22.4h\n" + "fcvtl v21.4s, v21.4h\n" + "ldr d20, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "fcvtl v20.4s, v20.4h\n" + "fcvtl v25.4s, v25.4h\n" + "ldr d19, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" + "fcvtl v19.4s, v19.4h\n" + "fcvtl v16.4s, v16.4h\n" + "zip1 v24.4s, v27.4s, v22.4s\n" + "zip1 v23.4s, v26.4s, v21.4s\n" + "subs %x[width], %x[width], #0x4\n" + "cmp %x[width], #0x4\n" + "zip1 v18.4s, v20.4s, v19.4s\n" + "zip1 v17.4s, v25.4s, v16.4s\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip2 v22.4s, v27.4s, v22.4s\n" + "zip2 v21.4s, v26.4s, v21.4s\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip2 v20.4s, v20.4s, v19.4s\n" + "zip2 v19.4s, v25.4s, v16.4s\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip1 v16.4s, v24.4s, v23.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v16.4s, v24.4s, v23.4s\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v17.4s, v18.4s, v17.4s\n" + "zip1 v16.4s, v22.4s, v21.4s\n" + "str q17, [%x[out_ptr], #0x30]\n" + "zip1 v18.4s, v20.4s, v19.4s\n" + "zip2 v17.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 6f\n" + "tbz %x[width], #1, 4f\n" + "ldr s28, [x28], #0x4\n" + "ldr s27, [x27], #0x4\n" + "mov x20, #0x2\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" + "ldr s24, [x24], #0x4\n" + "ldr s23, [x23], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" + "tbz %x[width], #0, 5f\n" + "ld1 { v28.h }[2], [x28]\n" + "ld1 { v27.h }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v26.h }[2], [x26]\n" + "ld1 { v25.h }[2], [x25]\n" + "ld1 { v24.h }[2], [x24]\n" + "ld1 { v23.h }[2], [x23]\n" + "ld1 { v22.h }[2], [x22]\n" + "ld1 { v21.h }[2], [x21]\n" + "b 5f\n" + "4:" // odd_loads_1_0 + "ldr h28, [x28, #0x0]\n" + "ldr h27, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr h26, [x26, #0x0]\n" + "ldr h25, [x25, #0x0]\n" + "ldr h24, [x24, #0x0]\n" + "ldr h23, [x23, #0x0]\n" + "ldr h22, [x22, #0x0]\n" + "ldr h21, [x21, #0x0]\n" + "5:" // Odd load end + "fcvtl v28.4s, v28.4h\n" + "fcvtl v27.4s, v27.4h\n" + "subs x20, x20, #0x1\n" + "fcvtl v26.4s, v26.4h\n" + "fcvtl v25.4s, v25.4h\n" + "fcvtl v24.4s, v24.4h\n" + "fcvtl v23.4s, v23.4h\n" + "fcvtl v22.4s, v22.4h\n" + "fcvtl v21.4s, v21.4h\n" + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 6f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 6f\n" + "zip2 v19.4s, v28.4s, v26.4s\n" + "zip2 v16.4s, v27.4s, v25.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v19.4s, v16.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "6:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp new file mode 100644 index 0000000000..80c387db47 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, false>( + float * &out_ptr, const float * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #2\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset], LSL #2\n" + "add x26, x26, %x[row_offset], LSL #2\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset], LSL #2\n" + "add x24, x24, %x[row_offset], LSL #2\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset], LSL #2\n" + "add x22, x22, %x[row_offset], LSL #2\n" + "add x21, x21, %x[row_offset], LSL #2\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x4\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q20, [x28], #0x10\n" + "ldr q18, [x27], #0x10\n" + "subs %x[width], %x[width], #0x4\n" + "cmp %x[width], #0x4\n" + "ldr q17, [x26], #0x10\n" + "ldr q16, [x25], #0x10\n" + "zip1 v25.4s, v20.4s, v17.4s\n" + "zip1 v24.4s, v18.4s, v16.4s\n" + "ldr q19, [x24], #0x10\n" + "ldr q23, [x23], #0x10\n" + "zip2 v22.4s, v20.4s, v17.4s\n" + "zip2 v21.4s, v18.4s, v16.4s\n" + "ldr q18, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v20.4s, v19.4s, v18.4s\n" + "zip1 v17.4s, v23.4s, v16.4s\n" + "zip2 v19.4s, v19.4s, v18.4s\n" + "zip2 v18.4s, v23.4s, v16.4s\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip1 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip2 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v16.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "zip2 v17.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 6f\n" + "tbz %x[width], #1, 4f\n" + "ldr d28, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" + "mov x20, #0x2\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "ldr d24, [x24], #0x8\n" + "ldr d23, [x23], #0x8\n" + "ldr d22, [x22], #0x8\n" + "ldr d21, [x21], #0x8\n" + "tbz %x[width], #0, 5f\n" + "ld1 { v28.s }[2], [x28]\n" + "ld1 { v27.s }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v26.s }[2], [x26]\n" + "ld1 { v25.s }[2], [x25]\n" + "ld1 { v24.s }[2], [x24]\n" + "ld1 { v23.s }[2], [x23]\n" + "ld1 { v22.s }[2], [x22]\n" + "ld1 { v21.s }[2], [x21]\n" + "b 5f\n" + "4:" // odd_loads_1_0 + "ldr s28, [x28, #0x0]\n" + "ldr s27, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr s26, [x26, #0x0]\n" + "ldr s25, [x25, #0x0]\n" + "ldr s24, [x24, #0x0]\n" + "ldr s23, [x23, #0x0]\n" + "ldr s22, [x22, #0x0]\n" + "ldr s21, [x21, #0x0]\n" + "5:" // Odd load end + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" + "subs x20, x20, #0x1\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 6f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 6f\n" + "zip2 v19.4s, v28.4s, v26.4s\n" + "zip2 v16.4s, v27.4s, v25.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v19.4s, v16.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "6:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp new file mode 100644 index 0000000000..8e06b7ecab --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, false>( + int16_t * &out_ptr, const int16_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset], LSL #1\n" + "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset], LSL #1\n" + "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset], LSL #1\n" + "add x22, x22, %x[row_offset], LSL #1\n" + "add x21, x21, %x[row_offset], LSL #1\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x8\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q25, [x28], #0x10\n" + "ldr q27, [x27], #0x10\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "ldr q26, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "ldr q21, [x24], #0x10\n" + "ldr q20, [x23], #0x10\n" + "zip1 v23.8h, v25.8h, v21.8h\n" + "zip1 v22.8h, v27.8h, v20.8h\n" + "ldr q17, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v19.8h, v26.8h, v17.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" + "zip2 v25.8h, v25.8h, v21.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip1 v24.8h, v23.8h, v19.8h\n" + "zip1 v17.8h, v22.8h, v18.8h\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip2 v23.8h, v23.8h, v19.8h\n" + "zip2 v19.8h, v22.8h, v18.8h\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip1 v22.8h, v25.8h, v21.8h\n" + "zip1 v18.8h, v20.8h, v16.8h\n" + "zip2 v21.8h, v25.8h, v21.8h\n" + "zip2 v20.8h, v20.8h, v16.8h\n" + "zip1 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v23.8h, v19.8h\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip1 v19.8h, v22.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v18.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "zip2 v16.8h, v21.8h, v20.8h\n" + "str q19, [%x[out_ptr], #0x40]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 8f\n" + "tbz %x[width], #2, 5f\n" + "ldr d30, [x28], #0x8\n" + "ldr d29, [x27], #0x8\n" + "ldr d28, [x26], #0x8\n" + "ldr d27, [x25], #0x8\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "ldr d24, [x22], #0x8\n" + "ldr d23, [x21], #0x8\n" + "tbz %x[width], #1, 4f\n" + "ld1 { v30.s }[2], [x28], #0x4\n" + "ld1 { v29.s }[2], [x27], #0x4\n" + "mov x20, #0x6\n" + "ld1 { v28.s }[2], [x26], #0x4\n" + "ld1 { v27.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x24], #0x4\n" + "ld1 { v25.s }[2], [x23], #0x4\n" + "ld1 { v24.s }[2], [x22], #0x4\n" + "ld1 { v23.s }[2], [x21], #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.h }[6], [x28]\n" + "ld1 { v29.h }[6], [x27]\n" + "mov x20, #0x7\n" + "ld1 { v28.h }[6], [x26]\n" + "ld1 { v27.h }[6], [x25]\n" + "ld1 { v26.h }[6], [x24]\n" + "ld1 { v25.h }[6], [x23]\n" + "ld1 { v24.h }[6], [x22]\n" + "ld1 { v23.h }[6], [x21]\n" + "b 7f\n" + "4:" // odd_loads_1_4 + "mov x20, #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.h }[4], [x28]\n" + "ld1 { v29.h }[4], [x27]\n" + "mov x20, #0x5\n" + "ld1 { v28.h }[4], [x26]\n" + "ld1 { v27.h }[4], [x25]\n" + "ld1 { v26.h }[4], [x24]\n" + "ld1 { v25.h }[4], [x23]\n" + "ld1 { v24.h }[4], [x22]\n" + "ld1 { v23.h }[4], [x21]\n" + "b 7f\n" + "5:" // odd_loads_2_0 + "tbz %x[width], #1, 6f\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "mov x20, #0x2\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.h }[2], [x28]\n" + "ld1 { v29.h }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v28.h }[2], [x26]\n" + "ld1 { v27.h }[2], [x25]\n" + "ld1 { v26.h }[2], [x24]\n" + "ld1 { v25.h }[2], [x23]\n" + "ld1 { v24.h }[2], [x22]\n" + "ld1 { v23.h }[2], [x21]\n" + "b 7f\n" + "6:" // odd_loads_1_0 + "ldr h30, [x28, #0x0]\n" + "ldr h29, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr h28, [x26, #0x0]\n" + "ldr h27, [x25, #0x0]\n" + "ldr h26, [x24, #0x0]\n" + "ldr h25, [x23, #0x0]\n" + "ldr h24, [x22, #0x0]\n" + "ldr h23, [x21, #0x0]\n" + "7:" // Odd load end + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "8:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + +template<> +void interleave_block<8, 1, VLType::None, false>( + uint16_t * &out_ptr, const uint16_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + int16_t * &out_cast = reinterpret_cast<int16_t * &>(out_ptr); + const int16_t * const * in_cast = reinterpret_cast<const int16_t * const *>(in); + + interleave_block<8, 1, VLType::None, false>(out_cast, in_cast, width, height, row_offset, false); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp new file mode 100644 index 0000000000..b91ae8a948 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, true>( + int16_t * &out_ptr, const int16_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "mov x20, #0x0\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "movi v2.8h, #0x0\n" + "movi v1.4s, #0x0\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "movi v0.4s, #0x0\n" + "add x28, x28, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x27, x27, %x[row_offset], LSL #1\n" + "add x26, x26, %x[row_offset], LSL #1\n" + "add x25, x25, %x[row_offset], LSL #1\n" + "add x24, x24, %x[row_offset], LSL #1\n" + "add x23, x23, %x[row_offset], LSL #1\n" + "add x22, x22, %x[row_offset], LSL #1\n" + "add x21, x21, %x[row_offset], LSL #1\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x20\n" + "ld1 { v1.4s }, [%x[out_ptr]]\n" + "ldr q0, [%x[out_ptr], #0x10]\n" + "2:" // first_pass + "cmp %x[width], #0x8\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x20, #0xe\n" + "ble 4f\n" + "saddw v1.4s, v1.4s, v2.4h\n" + "saddw2 v0.4s, v0.4s, v2.8h\n" + "mov x20, #0x0\n" + "movi v2.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr q31, [x28], #0x10\n" + "ldr q30, [x27], #0x10\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "ldr q29, [x26], #0x10\n" + "ldr q28, [x25], #0x10\n" + "add x20, x20, #0x1\n" + "ldr q27, [x24], #0x10\n" + "ldr q26, [x23], #0x10\n" + "zip1 v25.8h, v31.8h, v27.8h\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "ldr q24, [x22], #0x10\n" + "ldr q23, [x21], #0x10\n" + "zip1 v18.8h, v29.8h, v24.8h\n" + "zip1 v21.8h, v28.8h, v23.8h\n" + "zip1 v17.8h, v25.8h, v18.8h\n" + "zip1 v16.8h, v22.8h, v21.8h\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip1 v20.8h, v17.8h, v16.8h\n" + "add v2.8h, v2.8h, v20.8h\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip2 v19.8h, v17.8h, v16.8h\n" + "zip2 v18.8h, v25.8h, v18.8h\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "add v2.8h, v2.8h, v19.8h\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "zip2 v22.8h, v31.8h, v27.8h\n" + "str q20, [%x[out_ptr], #0x0]\n" + "zip2 v21.8h, v29.8h, v24.8h\n" + "zip2 v20.8h, v30.8h, v26.8h\n" + "str q19, [%x[out_ptr], #0x10]\n" + "zip2 v19.8h, v28.8h, v23.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x60]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 10f\n" + "tbz %x[width], #2, 7f\n" + "ldr d30, [x28], #0x8\n" + "ldr d29, [x27], #0x8\n" + "ldr d28, [x26], #0x8\n" + "ldr d27, [x25], #0x8\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "ldr d24, [x22], #0x8\n" + "ldr d23, [x21], #0x8\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v30.s }[2], [x28], #0x4\n" + "ld1 { v29.s }[2], [x27], #0x4\n" + "mov x20, #0x6\n" + "ld1 { v28.s }[2], [x26], #0x4\n" + "ld1 { v27.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x24], #0x4\n" + "ld1 { v25.s }[2], [x23], #0x4\n" + "ld1 { v24.s }[2], [x22], #0x4\n" + "ld1 { v23.s }[2], [x21], #0x4\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.h }[6], [x28]\n" + "ld1 { v29.h }[6], [x27]\n" + "mov x20, #0x7\n" + "ld1 { v28.h }[6], [x26]\n" + "ld1 { v27.h }[6], [x25]\n" + "ld1 { v26.h }[6], [x24]\n" + "ld1 { v25.h }[6], [x23]\n" + "ld1 { v24.h }[6], [x22]\n" + "ld1 { v23.h }[6], [x21]\n" + "b 9f\n" + "6:" // odd_loads_1_4 + "mov x20, #0x4\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.h }[4], [x28]\n" + "ld1 { v29.h }[4], [x27]\n" + "mov x20, #0x5\n" + "ld1 { v28.h }[4], [x26]\n" + "ld1 { v27.h }[4], [x25]\n" + "ld1 { v26.h }[4], [x24]\n" + "ld1 { v25.h }[4], [x23]\n" + "ld1 { v24.h }[4], [x22]\n" + "ld1 { v23.h }[4], [x21]\n" + "b 9f\n" + "7:" // odd_loads_2_0 + "tbz %x[width], #1, 8f\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "mov x20, #0x2\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.h }[2], [x28]\n" + "ld1 { v29.h }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v28.h }[2], [x26]\n" + "ld1 { v27.h }[2], [x25]\n" + "ld1 { v26.h }[2], [x24]\n" + "ld1 { v25.h }[2], [x23]\n" + "ld1 { v24.h }[2], [x22]\n" + "ld1 { v23.h }[2], [x21]\n" + "b 9f\n" + "8:" // odd_loads_1_0 + "ldr h30, [x28, #0x0]\n" + "ldr h29, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr h28, [x26, #0x0]\n" + "ldr h27, [x25, #0x0]\n" + "ldr h26, [x24, #0x0]\n" + "ldr h25, [x23, #0x0]\n" + "ldr h24, [x22, #0x0]\n" + "ldr h23, [x21, #0x0]\n" + "9:" // Odd load end + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "10:" // Odds skip + "saddw v1.4s, v1.4s, v2.4h\n" + "saddw2 v0.4s, v0.4s, v2.8h\n" + "str q1, [%x[out_ptr], #0x0]\n" + "str q0, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp new file mode 100644 index 0000000000..c41120c698 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, false>( + int16_t * &out_ptr, const int8_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset]\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x8\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr d25, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" + "sshll v25.8h, v25.8b, #0x0\n" + "sshll v27.8h, v27.8b, #0x0\n" + "ldr d26, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "sshll v26.8h, v26.8b, #0x0\n" + "sshll v24.8h, v24.8b, #0x0\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "sshll v21.8h, v21.8b, #0x0\n" + "sshll v20.8h, v20.8b, #0x0\n" + "ldr d17, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" + "sshll v17.8h, v17.8b, #0x0\n" + "sshll v16.8h, v16.8b, #0x0\n" + "zip1 v23.8h, v25.8h, v21.8h\n" + "zip1 v22.8h, v26.8h, v17.8h\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "zip1 v19.8h, v27.8h, v20.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip2 v25.8h, v25.8h, v21.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v24.8h, v23.8h, v22.8h\n" + "zip1 v17.8h, v19.8h, v18.8h\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip2 v23.8h, v23.8h, v22.8h\n" + "zip2 v19.8h, v19.8h, v18.8h\n" + "zip1 v22.8h, v25.8h, v21.8h\n" + "zip1 v18.8h, v20.8h, v16.8h\n" + "zip2 v21.8h, v25.8h, v21.8h\n" + "zip2 v20.8h, v20.8h, v16.8h\n" + "zip1 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v23.8h, v19.8h\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip1 v19.8h, v22.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v18.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "zip2 v16.8h, v21.8h, v20.8h\n" + "str q19, [%x[out_ptr], #0x40]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 8f\n" + "tbz %x[width], #2, 5f\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" + "tbz %x[width], #1, 4f\n" + "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x27], #0x2\n" + "mov x20, #0x6\n" + "ld1 { v28.h }[2], [x26], #0x2\n" + "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v26.h }[2], [x24], #0x2\n" + "ld1 { v25.h }[2], [x23], #0x2\n" + "ld1 { v24.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x27]\n" + "mov x20, #0x7\n" + "ld1 { v28.b }[6], [x26]\n" + "ld1 { v27.b }[6], [x25]\n" + "ld1 { v26.b }[6], [x24]\n" + "ld1 { v25.b }[6], [x23]\n" + "ld1 { v24.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" + "b 7f\n" + "4:" // odd_loads_1_4 + "mov x20, #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x27]\n" + "mov x20, #0x5\n" + "ld1 { v28.b }[4], [x26]\n" + "ld1 { v27.b }[4], [x25]\n" + "ld1 { v26.b }[4], [x24]\n" + "ld1 { v25.b }[4], [x23]\n" + "ld1 { v24.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" + "b 7f\n" + "5:" // odd_loads_2_0 + "tbz %x[width], #1, 6f\n" + "ldr h30, [x28], #0x2\n" + "ldr h29, [x27], #0x2\n" + "mov x20, #0x2\n" + "ldr h28, [x26], #0x2\n" + "ldr h27, [x25], #0x2\n" + "ldr h26, [x24], #0x2\n" + "ldr h25, [x23], #0x2\n" + "ldr h24, [x22], #0x2\n" + "ldr h23, [x21], #0x2\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v28.b }[2], [x26]\n" + "ld1 { v27.b }[2], [x25]\n" + "ld1 { v26.b }[2], [x24]\n" + "ld1 { v25.b }[2], [x23]\n" + "ld1 { v24.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" + "b 7f\n" + "6:" // odd_loads_1_0 + "ldr b30, [x28, #0x0]\n" + "ldr b29, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b28, [x26, #0x0]\n" + "ldr b27, [x25, #0x0]\n" + "ldr b26, [x24, #0x0]\n" + "ldr b25, [x23, #0x0]\n" + "ldr b24, [x22, #0x0]\n" + "ldr b23, [x21, #0x0]\n" + "7:" // Odd load end + "sshll v30.8h, v30.8b, #0x0\n" + "sshll v29.8h, v29.8b, #0x0\n" + "subs x20, x20, #0x1\n" + "sshll v28.8h, v28.8b, #0x0\n" + "sshll v27.8h, v27.8b, #0x0\n" + "sshll v26.8h, v26.8b, #0x0\n" + "sshll v25.8h, v25.8b, #0x0\n" + "sshll v24.8h, v24.8b, #0x0\n" + "sshll v23.8h, v23.8b, #0x0\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "8:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp new file mode 100644 index 0000000000..9ac7053ad8 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, true>( + int16_t * &out_ptr, const int8_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "mov x20, #0x0\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "movi v2.8h, #0x0\n" + "movi v1.4s, #0x0\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "movi v0.4s, #0x0\n" + "add x28, x28, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x20\n" + "ld1 { v1.4s }, [%x[out_ptr]]\n" + "ldr q0, [%x[out_ptr], #0x10]\n" + "2:" // first_pass + "cmp %x[width], #0x8\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x20, #0xe\n" + "ble 4f\n" + "saddw v1.4s, v1.4s, v2.4h\n" + "saddw2 v0.4s, v0.4s, v2.8h\n" + "mov x20, #0x0\n" + "movi v2.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr d31, [x28], #0x8\n" + "ldr d30, [x27], #0x8\n" + "sshll v31.8h, v31.8b, #0x0\n" + "sshll v30.8h, v30.8b, #0x0\n" + "ldr d29, [x26], #0x8\n" + "ldr d28, [x25], #0x8\n" + "sshll v29.8h, v29.8b, #0x0\n" + "sshll v28.8h, v28.8b, #0x0\n" + "ldr d27, [x24], #0x8\n" + "ldr d26, [x23], #0x8\n" + "sshll v27.8h, v27.8b, #0x0\n" + "sshll v26.8h, v26.8b, #0x0\n" + "ldr d25, [x22], #0x8\n" + "ldr d24, [x21], #0x8\n" + "sshll v25.8h, v25.8b, #0x0\n" + "sshll v24.8h, v24.8b, #0x0\n" + "zip1 v23.8h, v31.8h, v27.8h\n" + "zip1 v22.8h, v29.8h, v25.8h\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "zip1 v21.8h, v30.8h, v26.8h\n" + "zip1 v20.8h, v28.8h, v24.8h\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip1 v18.8h, v23.8h, v22.8h\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip2 v19.8h, v18.8h, v17.8h\n" + "zip2 v18.8h, v23.8h, v22.8h\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip2 v17.8h, v21.8h, v20.8h\n" + "add v2.8h, v2.8h, v19.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "zip2 v22.8h, v31.8h, v27.8h\n" + "str q19, [%x[out_ptr], #0x10]\n" + "zip2 v21.8h, v29.8h, v25.8h\n" + "zip2 v20.8h, v30.8h, v26.8h\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v19.8h, v28.8h, v24.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x60]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 10f\n" + "tbz %x[width], #2, 7f\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x27], #0x2\n" + "mov x20, #0x6\n" + "ld1 { v28.h }[2], [x26], #0x2\n" + "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v26.h }[2], [x24], #0x2\n" + "ld1 { v25.h }[2], [x23], #0x2\n" + "ld1 { v24.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x27]\n" + "mov x20, #0x7\n" + "ld1 { v28.b }[6], [x26]\n" + "ld1 { v27.b }[6], [x25]\n" + "ld1 { v26.b }[6], [x24]\n" + "ld1 { v25.b }[6], [x23]\n" + "ld1 { v24.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" + "b 9f\n" + "6:" // odd_loads_1_4 + "mov x20, #0x4\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x27]\n" + "mov x20, #0x5\n" + "ld1 { v28.b }[4], [x26]\n" + "ld1 { v27.b }[4], [x25]\n" + "ld1 { v26.b }[4], [x24]\n" + "ld1 { v25.b }[4], [x23]\n" + "ld1 { v24.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" + "b 9f\n" + "7:" // odd_loads_2_0 + "tbz %x[width], #1, 8f\n" + "ldr h30, [x28], #0x2\n" + "ldr h29, [x27], #0x2\n" + "mov x20, #0x2\n" + "ldr h28, [x26], #0x2\n" + "ldr h27, [x25], #0x2\n" + "ldr h26, [x24], #0x2\n" + "ldr h25, [x23], #0x2\n" + "ldr h24, [x22], #0x2\n" + "ldr h23, [x21], #0x2\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v28.b }[2], [x26]\n" + "ld1 { v27.b }[2], [x25]\n" + "ld1 { v26.b }[2], [x24]\n" + "ld1 { v25.b }[2], [x23]\n" + "ld1 { v24.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" + "b 9f\n" + "8:" // odd_loads_1_0 + "ldr b30, [x28, #0x0]\n" + "ldr b29, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b28, [x26, #0x0]\n" + "ldr b27, [x25, #0x0]\n" + "ldr b26, [x24, #0x0]\n" + "ldr b25, [x23, #0x0]\n" + "ldr b24, [x22, #0x0]\n" + "ldr b23, [x21, #0x0]\n" + "9:" // Odd load end + "sshll v30.8h, v30.8b, #0x0\n" + "sshll v29.8h, v29.8b, #0x0\n" + "subs x20, x20, #0x1\n" + "sshll v28.8h, v28.8b, #0x0\n" + "sshll v27.8h, v27.8b, #0x0\n" + "sshll v26.8h, v26.8b, #0x0\n" + "sshll v25.8h, v25.8b, #0x0\n" + "sshll v24.8h, v24.8b, #0x0\n" + "sshll v23.8h, v23.8b, #0x0\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "10:" // Odds skip + "saddw v1.4s, v1.4s, v2.4h\n" + "saddw2 v0.4s, v0.4s, v2.8h\n" + "str q1, [%x[out_ptr], #0x0]\n" + "str q0, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp new file mode 100644 index 0000000000..c01d980f49 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, true>( + uint16_t * &out_ptr, const uint16_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "mov x20, #0x0\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "movi v2.8h, #0x0\n" + "movi v1.4s, #0x0\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "movi v0.4s, #0x0\n" + "add x28, x28, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x27, x27, %x[row_offset], LSL #1\n" + "add x26, x26, %x[row_offset], LSL #1\n" + "add x25, x25, %x[row_offset], LSL #1\n" + "add x24, x24, %x[row_offset], LSL #1\n" + "add x23, x23, %x[row_offset], LSL #1\n" + "add x22, x22, %x[row_offset], LSL #1\n" + "add x21, x21, %x[row_offset], LSL #1\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x20\n" + "ld1 { v1.4s }, [%x[out_ptr]]\n" + "ldr q0, [%x[out_ptr], #0x10]\n" + "2:" // first_pass + "cmp %x[width], #0x8\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x20, #0xe\n" + "ble 4f\n" + "uaddw v1.4s, v1.4s, v2.4h\n" + "uaddw2 v0.4s, v0.4s, v2.8h\n" + "mov x20, #0x0\n" + "movi v2.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr q31, [x28], #0x10\n" + "ldr q30, [x27], #0x10\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "ldr q29, [x26], #0x10\n" + "ldr q28, [x25], #0x10\n" + "add x20, x20, #0x1\n" + "ldr q27, [x24], #0x10\n" + "ldr q26, [x23], #0x10\n" + "zip1 v25.8h, v31.8h, v27.8h\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "ldr q24, [x22], #0x10\n" + "ldr q23, [x21], #0x10\n" + "zip1 v18.8h, v29.8h, v24.8h\n" + "zip1 v21.8h, v28.8h, v23.8h\n" + "zip1 v17.8h, v25.8h, v18.8h\n" + "zip1 v16.8h, v22.8h, v21.8h\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip1 v20.8h, v17.8h, v16.8h\n" + "add v2.8h, v2.8h, v20.8h\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip2 v19.8h, v17.8h, v16.8h\n" + "zip2 v18.8h, v25.8h, v18.8h\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "add v2.8h, v2.8h, v19.8h\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "zip2 v22.8h, v31.8h, v27.8h\n" + "str q20, [%x[out_ptr], #0x0]\n" + "zip2 v21.8h, v29.8h, v24.8h\n" + "zip2 v20.8h, v30.8h, v26.8h\n" + "str q19, [%x[out_ptr], #0x10]\n" + "zip2 v19.8h, v28.8h, v23.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x60]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 10f\n" + "tbz %x[width], #2, 7f\n" + "ldr d30, [x28], #0x8\n" + "ldr d29, [x27], #0x8\n" + "ldr d28, [x26], #0x8\n" + "ldr d27, [x25], #0x8\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "ldr d24, [x22], #0x8\n" + "ldr d23, [x21], #0x8\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v30.s }[2], [x28], #0x4\n" + "ld1 { v29.s }[2], [x27], #0x4\n" + "mov x20, #0x6\n" + "ld1 { v28.s }[2], [x26], #0x4\n" + "ld1 { v27.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x24], #0x4\n" + "ld1 { v25.s }[2], [x23], #0x4\n" + "ld1 { v24.s }[2], [x22], #0x4\n" + "ld1 { v23.s }[2], [x21], #0x4\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.h }[6], [x28]\n" + "ld1 { v29.h }[6], [x27]\n" + "mov x20, #0x7\n" + "ld1 { v28.h }[6], [x26]\n" + "ld1 { v27.h }[6], [x25]\n" + "ld1 { v26.h }[6], [x24]\n" + "ld1 { v25.h }[6], [x23]\n" + "ld1 { v24.h }[6], [x22]\n" + "ld1 { v23.h }[6], [x21]\n" + "b 9f\n" + "6:" // odd_loads_1_4 + "mov x20, #0x4\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.h }[4], [x28]\n" + "ld1 { v29.h }[4], [x27]\n" + "mov x20, #0x5\n" + "ld1 { v28.h }[4], [x26]\n" + "ld1 { v27.h }[4], [x25]\n" + "ld1 { v26.h }[4], [x24]\n" + "ld1 { v25.h }[4], [x23]\n" + "ld1 { v24.h }[4], [x22]\n" + "ld1 { v23.h }[4], [x21]\n" + "b 9f\n" + "7:" // odd_loads_2_0 + "tbz %x[width], #1, 8f\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "mov x20, #0x2\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.h }[2], [x28]\n" + "ld1 { v29.h }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v28.h }[2], [x26]\n" + "ld1 { v27.h }[2], [x25]\n" + "ld1 { v26.h }[2], [x24]\n" + "ld1 { v25.h }[2], [x23]\n" + "ld1 { v24.h }[2], [x22]\n" + "ld1 { v23.h }[2], [x21]\n" + "b 9f\n" + "8:" // odd_loads_1_0 + "ldr h30, [x28, #0x0]\n" + "ldr h29, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr h28, [x26, #0x0]\n" + "ldr h27, [x25, #0x0]\n" + "ldr h26, [x24, #0x0]\n" + "ldr h25, [x23, #0x0]\n" + "ldr h24, [x22, #0x0]\n" + "ldr h23, [x21, #0x0]\n" + "9:" // Odd load end + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "10:" // Odds skip + "uaddw v1.4s, v1.4s, v2.4h\n" + "uaddw2 v0.4s, v0.4s, v2.8h\n" + "str q1, [%x[out_ptr], #0x0]\n" + "str q0, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp new file mode 100644 index 0000000000..d29a995b46 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, false>( + uint16_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset]\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x8\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr d25, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" + "ushll v25.8h, v25.8b, #0x0\n" + "ushll v27.8h, v27.8b, #0x0\n" + "ldr d26, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "ushll v26.8h, v26.8b, #0x0\n" + "ushll v24.8h, v24.8b, #0x0\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ushll v21.8h, v21.8b, #0x0\n" + "ushll v20.8h, v20.8b, #0x0\n" + "ldr d17, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" + "ushll v17.8h, v17.8b, #0x0\n" + "ushll v16.8h, v16.8b, #0x0\n" + "zip1 v23.8h, v25.8h, v21.8h\n" + "zip1 v22.8h, v26.8h, v17.8h\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "zip1 v19.8h, v27.8h, v20.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip2 v25.8h, v25.8h, v21.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v24.8h, v23.8h, v22.8h\n" + "zip1 v17.8h, v19.8h, v18.8h\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip2 v23.8h, v23.8h, v22.8h\n" + "zip2 v19.8h, v19.8h, v18.8h\n" + "zip1 v22.8h, v25.8h, v21.8h\n" + "zip1 v18.8h, v20.8h, v16.8h\n" + "zip2 v21.8h, v25.8h, v21.8h\n" + "zip2 v20.8h, v20.8h, v16.8h\n" + "zip1 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v24.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v23.8h, v19.8h\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip1 v19.8h, v22.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v18.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "zip2 v16.8h, v21.8h, v20.8h\n" + "str q19, [%x[out_ptr], #0x40]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 8f\n" + "tbz %x[width], #2, 5f\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" + "tbz %x[width], #1, 4f\n" + "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x27], #0x2\n" + "mov x20, #0x6\n" + "ld1 { v28.h }[2], [x26], #0x2\n" + "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v26.h }[2], [x24], #0x2\n" + "ld1 { v25.h }[2], [x23], #0x2\n" + "ld1 { v24.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x27]\n" + "mov x20, #0x7\n" + "ld1 { v28.b }[6], [x26]\n" + "ld1 { v27.b }[6], [x25]\n" + "ld1 { v26.b }[6], [x24]\n" + "ld1 { v25.b }[6], [x23]\n" + "ld1 { v24.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" + "b 7f\n" + "4:" // odd_loads_1_4 + "mov x20, #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x27]\n" + "mov x20, #0x5\n" + "ld1 { v28.b }[4], [x26]\n" + "ld1 { v27.b }[4], [x25]\n" + "ld1 { v26.b }[4], [x24]\n" + "ld1 { v25.b }[4], [x23]\n" + "ld1 { v24.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" + "b 7f\n" + "5:" // odd_loads_2_0 + "tbz %x[width], #1, 6f\n" + "ldr h30, [x28], #0x2\n" + "ldr h29, [x27], #0x2\n" + "mov x20, #0x2\n" + "ldr h28, [x26], #0x2\n" + "ldr h27, [x25], #0x2\n" + "ldr h26, [x24], #0x2\n" + "ldr h25, [x23], #0x2\n" + "ldr h24, [x22], #0x2\n" + "ldr h23, [x21], #0x2\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v28.b }[2], [x26]\n" + "ld1 { v27.b }[2], [x25]\n" + "ld1 { v26.b }[2], [x24]\n" + "ld1 { v25.b }[2], [x23]\n" + "ld1 { v24.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" + "b 7f\n" + "6:" // odd_loads_1_0 + "ldr b30, [x28, #0x0]\n" + "ldr b29, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b28, [x26, #0x0]\n" + "ldr b27, [x25, #0x0]\n" + "ldr b26, [x24, #0x0]\n" + "ldr b25, [x23, #0x0]\n" + "ldr b24, [x22, #0x0]\n" + "ldr b23, [x21, #0x0]\n" + "7:" // Odd load end + "ushll v30.8h, v30.8b, #0x0\n" + "ushll v29.8h, v29.8b, #0x0\n" + "subs x20, x20, #0x1\n" + "ushll v28.8h, v28.8b, #0x0\n" + "ushll v27.8h, v27.8b, #0x0\n" + "ushll v26.8h, v26.8b, #0x0\n" + "ushll v25.8h, v25.8b, #0x0\n" + "ushll v24.8h, v24.8b, #0x0\n" + "ushll v23.8h, v23.8b, #0x0\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 8f\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "8:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp new file mode 100644 index 0000000000..ae4bf9bf3b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 1, VLType::None, true>( + uint16_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "mov x20, #0x0\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "movi v2.8h, #0x0\n" + "movi v1.4s, #0x0\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "movi v0.4s, #0x0\n" + "add x28, x28, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x20\n" + "ld1 { v1.4s }, [%x[out_ptr]]\n" + "ldr q0, [%x[out_ptr], #0x10]\n" + "2:" // first_pass + "cmp %x[width], #0x8\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x20, #0xe\n" + "ble 4f\n" + "uaddw v1.4s, v1.4s, v2.4h\n" + "uaddw2 v0.4s, v0.4s, v2.8h\n" + "mov x20, #0x0\n" + "movi v2.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr d31, [x28], #0x8\n" + "ldr d30, [x27], #0x8\n" + "ushll v31.8h, v31.8b, #0x0\n" + "ushll v30.8h, v30.8b, #0x0\n" + "ldr d29, [x26], #0x8\n" + "ldr d28, [x25], #0x8\n" + "ushll v29.8h, v29.8b, #0x0\n" + "ushll v28.8h, v28.8b, #0x0\n" + "ldr d27, [x24], #0x8\n" + "ldr d26, [x23], #0x8\n" + "ushll v27.8h, v27.8b, #0x0\n" + "ushll v26.8h, v26.8b, #0x0\n" + "ldr d25, [x22], #0x8\n" + "ldr d24, [x21], #0x8\n" + "ushll v25.8h, v25.8b, #0x0\n" + "ushll v24.8h, v24.8b, #0x0\n" + "zip1 v23.8h, v31.8h, v27.8h\n" + "zip1 v22.8h, v29.8h, v25.8h\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "zip1 v21.8h, v30.8h, v26.8h\n" + "zip1 v20.8h, v28.8h, v24.8h\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip1 v18.8h, v23.8h, v22.8h\n" + "zip1 v17.8h, v21.8h, v20.8h\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip2 v19.8h, v18.8h, v17.8h\n" + "zip2 v18.8h, v23.8h, v22.8h\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip2 v17.8h, v21.8h, v20.8h\n" + "add v2.8h, v2.8h, v19.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "zip2 v22.8h, v31.8h, v27.8h\n" + "str q19, [%x[out_ptr], #0x10]\n" + "zip2 v21.8h, v29.8h, v25.8h\n" + "zip2 v20.8h, v30.8h, v26.8h\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v19.8h, v28.8h, v24.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "add v2.8h, v2.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x60]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 10f\n" + "tbz %x[width], #2, 7f\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x27], #0x2\n" + "mov x20, #0x6\n" + "ld1 { v28.h }[2], [x26], #0x2\n" + "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v26.h }[2], [x24], #0x2\n" + "ld1 { v25.h }[2], [x23], #0x2\n" + "ld1 { v24.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x27]\n" + "mov x20, #0x7\n" + "ld1 { v28.b }[6], [x26]\n" + "ld1 { v27.b }[6], [x25]\n" + "ld1 { v26.b }[6], [x24]\n" + "ld1 { v25.b }[6], [x23]\n" + "ld1 { v24.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" + "b 9f\n" + "6:" // odd_loads_1_4 + "mov x20, #0x4\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x27]\n" + "mov x20, #0x5\n" + "ld1 { v28.b }[4], [x26]\n" + "ld1 { v27.b }[4], [x25]\n" + "ld1 { v26.b }[4], [x24]\n" + "ld1 { v25.b }[4], [x23]\n" + "ld1 { v24.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" + "b 9f\n" + "7:" // odd_loads_2_0 + "tbz %x[width], #1, 8f\n" + "ldr h30, [x28], #0x2\n" + "ldr h29, [x27], #0x2\n" + "mov x20, #0x2\n" + "ldr h28, [x26], #0x2\n" + "ldr h27, [x25], #0x2\n" + "ldr h26, [x24], #0x2\n" + "ldr h25, [x23], #0x2\n" + "ldr h24, [x22], #0x2\n" + "ldr h23, [x21], #0x2\n" + "tbz %x[width], #0, 9f\n" + "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v28.b }[2], [x26]\n" + "ld1 { v27.b }[2], [x25]\n" + "ld1 { v26.b }[2], [x24]\n" + "ld1 { v25.b }[2], [x23]\n" + "ld1 { v24.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" + "b 9f\n" + "8:" // odd_loads_1_0 + "ldr b30, [x28, #0x0]\n" + "ldr b29, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b28, [x26, #0x0]\n" + "ldr b27, [x25, #0x0]\n" + "ldr b26, [x24, #0x0]\n" + "ldr b25, [x23, #0x0]\n" + "ldr b24, [x22, #0x0]\n" + "ldr b23, [x21, #0x0]\n" + "9:" // Odd load end + "ushll v30.8h, v30.8b, #0x0\n" + "ushll v29.8h, v29.8b, #0x0\n" + "subs x20, x20, #0x1\n" + "ushll v28.8h, v28.8b, #0x0\n" + "ushll v27.8h, v27.8b, #0x0\n" + "ushll v26.8h, v26.8b, #0x0\n" + "ushll v25.8h, v25.8b, #0x0\n" + "ushll v24.8h, v24.8b, #0x0\n" + "ushll v23.8h, v23.8b, #0x0\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "subs x20, x20, #0x1\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "beq 10f\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" + "add %x[out_ptr], %x[out_ptr], #0x10\n" + "10:" // Odds skip + "uaddw v1.4s, v1.4s, v2.4h\n" + "uaddw2 v0.4s, v0.4s, v2.8h\n" + "str q1, [%x[out_ptr], #0x0]\n" + "str q0, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp new file mode 100644 index 0000000000..43d9d20c10 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 2, VLType::None, false>( + bfloat16 * &out_ptr, const bfloat16 * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset], LSL #1\n" + "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset], LSL #1\n" + "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset], LSL #1\n" + "add x22, x22, %x[row_offset], LSL #1\n" + "add x21, x21, %x[row_offset], LSL #1\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x8\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q20, [x28], #0x10\n" + "ldr q18, [x27], #0x10\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "ldr q17, [x26], #0x10\n" + "ldr q16, [x25], #0x10\n" + "zip1 v25.4s, v20.4s, v17.4s\n" + "zip1 v24.4s, v18.4s, v16.4s\n" + "ldr q19, [x24], #0x10\n" + "ldr q23, [x23], #0x10\n" + "zip2 v22.4s, v20.4s, v17.4s\n" + "zip2 v21.4s, v18.4s, v16.4s\n" + "ldr q18, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v20.4s, v19.4s, v18.4s\n" + "zip1 v17.4s, v23.4s, v16.4s\n" + "zip2 v19.4s, v19.4s, v18.4s\n" + "zip2 v18.4s, v23.4s, v16.4s\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip1 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip2 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v16.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "zip2 v17.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 8f\n" + "tbz %x[width], #2, 5f\n" + "ldr d28, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "ldr d24, [x24], #0x8\n" + "ldr d23, [x23], #0x8\n" + "ldr d22, [x22], #0x8\n" + "ldr d21, [x21], #0x8\n" + "tbz %x[width], #1, 4f\n" + "ld1 { v28.s }[2], [x28], #0x4\n" + "ld1 { v27.s }[2], [x27], #0x4\n" + "mov x20, #0x3\n" + "ld1 { v26.s }[2], [x26], #0x4\n" + "ld1 { v25.s }[2], [x25], #0x4\n" + "ld1 { v24.s }[2], [x24], #0x4\n" + "ld1 { v23.s }[2], [x23], #0x4\n" + "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v21.s }[2], [x21], #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v28.h }[6], [x28]\n" + "ld1 { v27.h }[6], [x27]\n" + "mov x20, #0x4\n" + "ld1 { v26.h }[6], [x26]\n" + "ld1 { v25.h }[6], [x25]\n" + "ld1 { v24.h }[6], [x24]\n" + "ld1 { v23.h }[6], [x23]\n" + "ld1 { v22.h }[6], [x22]\n" + "ld1 { v21.h }[6], [x21]\n" + "b 7f\n" + "4:" // odd_loads_1_4 + "mov x20, #0x2\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v28.h }[4], [x28]\n" + "ld1 { v27.h }[4], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v26.h }[4], [x26]\n" + "ld1 { v25.h }[4], [x25]\n" + "ld1 { v24.h }[4], [x24]\n" + "ld1 { v23.h }[4], [x23]\n" + "ld1 { v22.h }[4], [x22]\n" + "ld1 { v21.h }[4], [x21]\n" + "b 7f\n" + "5:" // odd_loads_2_0 + "tbz %x[width], #1, 6f\n" + "ldr s28, [x28], #0x4\n" + "ldr s27, [x27], #0x4\n" + "mov x20, #0x1\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" + "ldr s24, [x24], #0x4\n" + "ldr s23, [x23], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v28.h }[2], [x28]\n" + "ld1 { v27.h }[2], [x27]\n" + "mov x20, #0x2\n" + "ld1 { v26.h }[2], [x26]\n" + "ld1 { v25.h }[2], [x25]\n" + "ld1 { v24.h }[2], [x24]\n" + "ld1 { v23.h }[2], [x23]\n" + "ld1 { v22.h }[2], [x22]\n" + "ld1 { v21.h }[2], [x21]\n" + "b 7f\n" + "6:" // odd_loads_1_0 + "ldr h28, [x28, #0x0]\n" + "ldr h27, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr h26, [x26, #0x0]\n" + "ldr h25, [x25, #0x0]\n" + "ldr h24, [x24, #0x0]\n" + "ldr h23, [x23, #0x0]\n" + "ldr h22, [x22, #0x0]\n" + "ldr h21, [x21, #0x0]\n" + "7:" // Odd load end + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" + "subs x20, x20, #0x1\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 8f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 8f\n" + "zip2 v20.4s, v28.4s, v26.4s\n" + "zip2 v19.4s, v27.4s, v25.4s\n" + "subs x20, x20, #0x1\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 8f\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "8:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp new file mode 100644 index 0000000000..3ec03370a0 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 2, VLType::None, false>( + float * &out_ptr, const float * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #2\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset], LSL #2\n" + "add x26, x26, %x[row_offset], LSL #2\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset], LSL #2\n" + "add x24, x24, %x[row_offset], LSL #2\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset], LSL #2\n" + "add x22, x22, %x[row_offset], LSL #2\n" + "add x21, x21, %x[row_offset], LSL #2\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x4\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q20, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" + "subs %x[width], %x[width], #0x4\n" + "cmp %x[width], #0x4\n" + "ldr q25, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "zip1 v16.2d, v20.2d, v19.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" + "ldr q23, [x24], #0x10\n" + "ldr q22, [x23], #0x10\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip2 v21.2d, v20.2d, v19.2d\n" + "ldr q20, [x22], #0x10\n" + "ldr q19, [x21], #0x10\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v20.2d, v19.2d\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "str q18, [%x[out_ptr], #0x10]\n" + "zip2 v18.2d, v25.2d, v24.2d\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip2 v17.2d, v23.2d, v22.2d\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip2 v16.2d, v20.2d, v19.2d\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "str q21, [%x[out_ptr], #0x40]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 6f\n" + "tbz %x[width], #1, 4f\n" + "ldr d25, [x28], #0x8\n" + "ldr d24, [x27], #0x8\n" + "mov x20, #0x1\n" + "ldr d23, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d19, [x22], #0x8\n" + "ldr d18, [x21], #0x8\n" + "tbz %x[width], #0, 5f\n" + "ld1 { v25.s }[2], [x28]\n" + "ld1 { v24.s }[2], [x27]\n" + "mov x20, #0x2\n" + "ld1 { v23.s }[2], [x26]\n" + "ld1 { v22.s }[2], [x25]\n" + "ld1 { v21.s }[2], [x24]\n" + "ld1 { v20.s }[2], [x23]\n" + "ld1 { v19.s }[2], [x22]\n" + "ld1 { v18.s }[2], [x21]\n" + "b 5f\n" + "4:" // odd_loads_1_0 + "ldr s25, [x28, #0x0]\n" + "ldr s24, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr s23, [x26, #0x0]\n" + "ldr s22, [x25, #0x0]\n" + "ldr s21, [x24, #0x0]\n" + "ldr s20, [x23, #0x0]\n" + "ldr s19, [x22, #0x0]\n" + "ldr s18, [x21, #0x0]\n" + "5:" // Odd load end + "subs x20, x20, #0x1\n" + "zip1 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.2d, v21.2d, v20.2d\n" + "zip1 v16.2d, v19.2d, v18.2d\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "beq 6f\n" + "zip2 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v17.2d, v21.2d, v20.2d\n" + "zip2 v16.2d, v19.2d, v18.2d\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "6:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp new file mode 100644 index 0000000000..e9799f87a9 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 4, VLType::None, false>( + bfloat16 * &out_ptr, const bfloat16 * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #1\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset], LSL #1\n" + "add x26, x26, %x[row_offset], LSL #1\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset], LSL #1\n" + "add x24, x24, %x[row_offset], LSL #1\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset], LSL #1\n" + "add x22, x22, %x[row_offset], LSL #1\n" + "add x21, x21, %x[row_offset], LSL #1\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x8\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q20, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" + "subs %x[width], %x[width], #0x8\n" + "cmp %x[width], #0x8\n" + "ldr q25, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "zip1 v16.2d, v20.2d, v19.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" + "ldr q23, [x24], #0x10\n" + "ldr q22, [x23], #0x10\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip2 v21.2d, v20.2d, v19.2d\n" + "ldr q20, [x22], #0x10\n" + "ldr q19, [x21], #0x10\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v20.2d, v19.2d\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "str q18, [%x[out_ptr], #0x10]\n" + "zip2 v18.2d, v25.2d, v24.2d\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip2 v17.2d, v23.2d, v22.2d\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip2 v16.2d, v20.2d, v19.2d\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "str q21, [%x[out_ptr], #0x40]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 8f\n" + "tbz %x[width], #2, 5f\n" + "ldr d25, [x28], #0x8\n" + "ldr d24, [x27], #0x8\n" + "ldr d23, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d19, [x22], #0x8\n" + "ldr d18, [x21], #0x8\n" + "tbz %x[width], #1, 4f\n" + "ld1 { v25.s }[2], [x28], #0x4\n" + "ld1 { v24.s }[2], [x27], #0x4\n" + "mov x20, #0x2\n" + "ld1 { v23.s }[2], [x26], #0x4\n" + "ld1 { v22.s }[2], [x25], #0x4\n" + "ld1 { v21.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v19.s }[2], [x22], #0x4\n" + "ld1 { v18.s }[2], [x21], #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v25.h }[6], [x28]\n" + "ld1 { v24.h }[6], [x27]\n" + "ld1 { v23.h }[6], [x26]\n" + "ld1 { v22.h }[6], [x25]\n" + "ld1 { v21.h }[6], [x24]\n" + "ld1 { v20.h }[6], [x23]\n" + "ld1 { v19.h }[6], [x22]\n" + "ld1 { v18.h }[6], [x21]\n" + "b 7f\n" + "4:" // odd_loads_1_4 + "mov x20, #0x1\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v25.h }[4], [x28]\n" + "ld1 { v24.h }[4], [x27]\n" + "mov x20, #0x2\n" + "ld1 { v23.h }[4], [x26]\n" + "ld1 { v22.h }[4], [x25]\n" + "ld1 { v21.h }[4], [x24]\n" + "ld1 { v20.h }[4], [x23]\n" + "ld1 { v19.h }[4], [x22]\n" + "ld1 { v18.h }[4], [x21]\n" + "b 7f\n" + "5:" // odd_loads_2_0 + "tbz %x[width], #1, 6f\n" + "ldr s25, [x28], #0x4\n" + "ldr s24, [x27], #0x4\n" + "mov x20, #0x1\n" + "ldr s23, [x26], #0x4\n" + "ldr s22, [x25], #0x4\n" + "ldr s21, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s19, [x22], #0x4\n" + "ldr s18, [x21], #0x4\n" + "tbz %x[width], #0, 7f\n" + "ld1 { v25.h }[2], [x28]\n" + "ld1 { v24.h }[2], [x27]\n" + "ld1 { v23.h }[2], [x26]\n" + "ld1 { v22.h }[2], [x25]\n" + "ld1 { v21.h }[2], [x24]\n" + "ld1 { v20.h }[2], [x23]\n" + "ld1 { v19.h }[2], [x22]\n" + "ld1 { v18.h }[2], [x21]\n" + "b 7f\n" + "6:" // odd_loads_1_0 + "ldr h25, [x28, #0x0]\n" + "ldr h24, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr h23, [x26, #0x0]\n" + "ldr h22, [x25, #0x0]\n" + "ldr h21, [x24, #0x0]\n" + "ldr h20, [x23, #0x0]\n" + "ldr h19, [x22, #0x0]\n" + "ldr h18, [x21, #0x0]\n" + "7:" // Odd load end + "subs x20, x20, #0x1\n" + "zip1 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.2d, v21.2d, v20.2d\n" + "zip1 v16.2d, v19.2d, v18.2d\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "beq 8f\n" + "zip2 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v17.2d, v21.2d, v20.2d\n" + "zip2 v16.2d, v19.2d, v18.2d\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "8:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp new file mode 100644 index 0000000000..730bfd6342 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 4, VLType::None, false>( + bfloat16 * &out_ptr, const float * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset], LSL #2\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset], LSL #2\n" + "add x26, x26, %x[row_offset], LSL #2\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset], LSL #2\n" + "add x24, x24, %x[row_offset], LSL #2\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset], LSL #2\n" + "add x22, x22, %x[row_offset], LSL #2\n" + "add x21, x21, %x[row_offset], LSL #2\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x4\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q17, [x28], #0x10\n" + "ldr q16, [x26], #0x10\n" + ".inst 0x0ea16a37 // bfcvtn v23.4h, v17.4s\n" + ".inst 0x0ea16a16 // bfcvtn v22.4h, v16.4s\n" + "ldr q17, [x24], #0x10\n" + "ldr q16, [x22], #0x10\n" + ".inst 0x0ea16a35 // bfcvtn v21.4h, v17.4s\n" + ".inst 0x0ea16a14 // bfcvtn v20.4h, v16.4s\n" + "ldr q19, [x27], #0x10\n" + "ldr q18, [x25], #0x10\n" + "subs %x[width], %x[width], #0x4\n" + "cmp %x[width], #0x4\n" + "ldr q17, [x23], #0x10\n" + "ldr q16, [x21], #0x10\n" + ".inst 0x4ea16a77 // bfcvtn2 v23.8h, v19.4s\n" + ".inst 0x4ea16a56 // bfcvtn2 v22.8h, v18.4s\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + ".inst 0x4ea16a35 // bfcvtn2 v21.8h, v17.4s\n" + ".inst 0x4ea16a14 // bfcvtn2 v20.8h, v16.4s\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "str q23, [%x[out_ptr], #0x0]\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "str q22, [%x[out_ptr], #0x10]\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "str q21, [%x[out_ptr], #0x20]\n" + "str q20, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 6f\n" + "tbz %x[width], #1, 4f\n" + "ldr d19, [x28], #0x8\n" + "ldr d23, [x27], #0x8\n" + "mov x20, #0x1\n" + "ldr d18, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d17, [x24], #0x8\n" + "ldr d21, [x23], #0x8\n" + "ldr d16, [x22], #0x8\n" + "ldr d20, [x21], #0x8\n" + "tbz %x[width], #0, 5f\n" + "ld1 { v19.s }[2], [x28]\n" + "ld1 { v23.s }[2], [x27]\n" + "ld1 { v18.s }[2], [x26]\n" + "ld1 { v22.s }[2], [x25]\n" + "ld1 { v17.s }[2], [x24]\n" + "ld1 { v21.s }[2], [x23]\n" + "ld1 { v16.s }[2], [x22]\n" + "ld1 { v20.s }[2], [x21]\n" + "b 5f\n" + "4:" // odd_loads_1_0 + "ldr s19, [x28, #0x0]\n" + "ldr s23, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr s18, [x26, #0x0]\n" + "ldr s22, [x25, #0x0]\n" + "ldr s17, [x24, #0x0]\n" + "ldr s21, [x23, #0x0]\n" + "ldr s16, [x22, #0x0]\n" + "ldr s20, [x21, #0x0]\n" + "5:" // Odd load end + ".inst 0x0ea16a73 // bfcvtn v19.4h, v19.4s\n" + ".inst 0x0ea16a52 // bfcvtn v18.4h, v18.4s\n" + ".inst 0x0ea16a31 // bfcvtn v17.4h, v17.4s\n" + ".inst 0x0ea16a10 // bfcvtn v16.4h, v16.4s\n" + ".inst 0x4ea16af3 // bfcvtn2 v19.8h, v23.4s\n" + ".inst 0x4ea16ad2 // bfcvtn2 v18.8h, v22.4s\n" + "str q19, [%x[out_ptr], #0x0]\n" + ".inst 0x4ea16ab1 // bfcvtn2 v17.8h, v21.4s\n" + ".inst 0x4ea16a90 // bfcvtn2 v16.8h, v20.4s\n" + "str q18, [%x[out_ptr], #0x10]\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "6:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp new file mode 100644 index 0000000000..15d8ddbe53 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 4, VLType::None, false>( + int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset]\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x10\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q20, [x28], #0x10\n" + "ldr q18, [x27], #0x10\n" + "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" + "ldr q17, [x26], #0x10\n" + "ldr q16, [x25], #0x10\n" + "zip1 v25.4s, v20.4s, v17.4s\n" + "zip1 v24.4s, v18.4s, v16.4s\n" + "ldr q19, [x24], #0x10\n" + "ldr q23, [x23], #0x10\n" + "zip2 v22.4s, v20.4s, v17.4s\n" + "zip2 v21.4s, v18.4s, v16.4s\n" + "ldr q18, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v20.4s, v19.4s, v18.4s\n" + "zip1 v17.4s, v23.4s, v16.4s\n" + "zip2 v19.4s, v19.4s, v18.4s\n" + "zip2 v18.4s, v23.4s, v16.4s\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip1 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip1 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "zip2 v16.4s, v25.4s, v24.4s\n" + "str q16, [%x[out_ptr], #0x20]\n" + "zip2 v16.4s, v20.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip1 v16.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x40]\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "zip2 v17.4s, v22.4s, v21.4s\n" + "str q16, [%x[out_ptr], #0x50]\n" + "zip2 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 12f\n" + "tbz %x[width], #3, 7f\n" + "ldr d28, [x28], #0x8\n" + "ldr d27, [x27], #0x8\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "ldr d24, [x24], #0x8\n" + "ldr d23, [x23], #0x8\n" + "ldr d22, [x22], #0x8\n" + "ldr d21, [x21], #0x8\n" + "tbz %x[width], #2, 5f\n" + "ld1 { v28.s }[2], [x28], #0x4\n" + "ld1 { v27.s }[2], [x27], #0x4\n" + "ld1 { v26.s }[2], [x26], #0x4\n" + "ld1 { v25.s }[2], [x25], #0x4\n" + "ld1 { v24.s }[2], [x24], #0x4\n" + "ld1 { v23.s }[2], [x23], #0x4\n" + "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v21.s }[2], [x21], #0x4\n" + "tbz %x[width], #1, 4f\n" + "ld1 { v28.h }[6], [x28], #0x2\n" + "ld1 { v27.h }[6], [x27], #0x2\n" + "mov x20, #0x4\n" + "ld1 { v26.h }[6], [x26], #0x2\n" + "ld1 { v25.h }[6], [x25], #0x2\n" + "ld1 { v24.h }[6], [x24], #0x2\n" + "ld1 { v23.h }[6], [x23], #0x2\n" + "ld1 { v22.h }[6], [x22], #0x2\n" + "ld1 { v21.h }[6], [x21], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v28.b }[14], [x28]\n" + "ld1 { v27.b }[14], [x27]\n" + "ld1 { v26.b }[14], [x26]\n" + "ld1 { v25.b }[14], [x25]\n" + "ld1 { v24.b }[14], [x24]\n" + "ld1 { v23.b }[14], [x23]\n" + "ld1 { v22.b }[14], [x22]\n" + "ld1 { v21.b }[14], [x21]\n" + "b 11f\n" + "4:" // odd_loads_1_12 + "mov x20, #0x3\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v28.b }[12], [x28]\n" + "ld1 { v27.b }[12], [x27]\n" + "mov x20, #0x4\n" + "ld1 { v26.b }[12], [x26]\n" + "ld1 { v25.b }[12], [x25]\n" + "ld1 { v24.b }[12], [x24]\n" + "ld1 { v23.b }[12], [x23]\n" + "ld1 { v22.b }[12], [x22]\n" + "ld1 { v21.b }[12], [x21]\n" + "b 11f\n" + "5:" // odd_loads_2_8 + "tbz %x[width], #1, 6f\n" + "ld1 { v28.h }[4], [x28], #0x2\n" + "ld1 { v27.h }[4], [x27], #0x2\n" + "mov x20, #0x3\n" + "ld1 { v26.h }[4], [x26], #0x2\n" + "ld1 { v25.h }[4], [x25], #0x2\n" + "ld1 { v24.h }[4], [x24], #0x2\n" + "ld1 { v23.h }[4], [x23], #0x2\n" + "ld1 { v22.h }[4], [x22], #0x2\n" + "ld1 { v21.h }[4], [x21], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v28.b }[10], [x28]\n" + "ld1 { v27.b }[10], [x27]\n" + "ld1 { v26.b }[10], [x26]\n" + "ld1 { v25.b }[10], [x25]\n" + "ld1 { v24.b }[10], [x24]\n" + "ld1 { v23.b }[10], [x23]\n" + "ld1 { v22.b }[10], [x22]\n" + "ld1 { v21.b }[10], [x21]\n" + "b 11f\n" + "6:" // odd_loads_1_8 + "mov x20, #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v28.b }[8], [x28]\n" + "ld1 { v27.b }[8], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v26.b }[8], [x26]\n" + "ld1 { v25.b }[8], [x25]\n" + "ld1 { v24.b }[8], [x24]\n" + "ld1 { v23.b }[8], [x23]\n" + "ld1 { v22.b }[8], [x22]\n" + "ld1 { v21.b }[8], [x21]\n" + "b 11f\n" + "7:" // odd_loads_4_0 + "tbz %x[width], #2, 9f\n" + "ldr s28, [x28], #0x4\n" + "ldr s27, [x27], #0x4\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" + "ldr s24, [x24], #0x4\n" + "ldr s23, [x23], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" + "tbz %x[width], #1, 8f\n" + "ld1 { v28.h }[2], [x28], #0x2\n" + "ld1 { v27.h }[2], [x27], #0x2\n" + "mov x20, #0x2\n" + "ld1 { v26.h }[2], [x26], #0x2\n" + "ld1 { v25.h }[2], [x25], #0x2\n" + "ld1 { v24.h }[2], [x24], #0x2\n" + "ld1 { v23.h }[2], [x23], #0x2\n" + "ld1 { v22.h }[2], [x22], #0x2\n" + "ld1 { v21.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v28.b }[6], [x28]\n" + "ld1 { v27.b }[6], [x27]\n" + "ld1 { v26.b }[6], [x26]\n" + "ld1 { v25.b }[6], [x25]\n" + "ld1 { v24.b }[6], [x24]\n" + "ld1 { v23.b }[6], [x23]\n" + "ld1 { v22.b }[6], [x22]\n" + "ld1 { v21.b }[6], [x21]\n" + "b 11f\n" + "8:" // odd_loads_1_4 + "mov x20, #0x1\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v28.b }[4], [x28]\n" + "ld1 { v27.b }[4], [x27]\n" + "mov x20, #0x2\n" + "ld1 { v26.b }[4], [x26]\n" + "ld1 { v25.b }[4], [x25]\n" + "ld1 { v24.b }[4], [x24]\n" + "ld1 { v23.b }[4], [x23]\n" + "ld1 { v22.b }[4], [x22]\n" + "ld1 { v21.b }[4], [x21]\n" + "b 11f\n" + "9:" // odd_loads_2_0 + "tbz %x[width], #1, 10f\n" + "ldr h28, [x28], #0x2\n" + "ldr h27, [x27], #0x2\n" + "mov x20, #0x1\n" + "ldr h26, [x26], #0x2\n" + "ldr h25, [x25], #0x2\n" + "ldr h24, [x24], #0x2\n" + "ldr h23, [x23], #0x2\n" + "ldr h22, [x22], #0x2\n" + "ldr h21, [x21], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v28.b }[2], [x28]\n" + "ld1 { v27.b }[2], [x27]\n" + "ld1 { v26.b }[2], [x26]\n" + "ld1 { v25.b }[2], [x25]\n" + "ld1 { v24.b }[2], [x24]\n" + "ld1 { v23.b }[2], [x23]\n" + "ld1 { v22.b }[2], [x22]\n" + "ld1 { v21.b }[2], [x21]\n" + "b 11f\n" + "10:" // odd_loads_1_0 + "ldr b28, [x28, #0x0]\n" + "ldr b27, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b26, [x26, #0x0]\n" + "ldr b25, [x25, #0x0]\n" + "ldr b24, [x24, #0x0]\n" + "ldr b23, [x23, #0x0]\n" + "ldr b22, [x22, #0x0]\n" + "ldr b21, [x21, #0x0]\n" + "11:" // Odd load end + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" + "subs x20, x20, #0x1\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 12f\n" + "subs x20, x20, #0x1\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 12f\n" + "zip2 v20.4s, v28.4s, v26.4s\n" + "zip2 v19.4s, v27.4s, v25.4s\n" + "subs x20, x20, #0x1\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 12f\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "12:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + +template<> +void interleave_block<8, 4, VLType::None, false>( + uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + int8_t * &out_cast = reinterpret_cast<int8_t * &>(out_ptr); + const int8_t * const * in_cast = reinterpret_cast<const int8_t * const *>(in); + + interleave_block<8, 4, VLType::None, false>(out_cast, in_cast, width, height, row_offset, false); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp new file mode 100644 index 0000000000..6c41b5fdfb --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp @@ -0,0 +1,370 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 4, VLType::None, true>( + int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "mov x20, #0x0\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "movi v2.8h, #0x0\n" + "movi v1.8h, #0x0\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "movi v0.4s, #0x0\n" + "movi v31.4s, #0x0\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x28, x28, %x[row_offset]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x20\n" + "ld1 { v0.4s }, [%x[out_ptr]]\n" + "ldr q31, [%x[out_ptr], #0x10]\n" + "2:" // first_pass + "cmp %x[width], #0x10\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x20, #0x1e\n" + "ble 4f\n" + "sadalp v0.4s, v2.8h\n" + "movi v2.8h, #0x0\n" + "mov x20, #0x0\n" + "sadalp v31.4s, v1.8h\n" + "movi v1.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr q30, [x28], #0x10\n" + "ldr q29, [x27], #0x10\n" + "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" + "ldr q28, [x26], #0x10\n" + "ldr q27, [x25], #0x10\n" + "zip1 v22.4s, v30.4s, v28.4s\n" + "zip1 v21.4s, v29.4s, v27.4s\n" + "ldr q20, [x24], #0x10\n" + "ldr q26, [x23], #0x10\n" + "zip1 v25.4s, v22.4s, v21.4s\n" + "sadalp v2.8h, v25.16b\n" + "ldr q19, [x22], #0x10\n" + "ldr q18, [x21], #0x10\n" + "zip1 v17.4s, v20.4s, v19.4s\n" + "zip1 v16.4s, v26.4s, v18.4s\n" + "zip1 v24.4s, v17.4s, v16.4s\n" + "sadalp v1.8h, v24.16b\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip2 v23.4s, v22.4s, v21.4s\n" + "zip2 v22.4s, v17.4s, v16.4s\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip2 v21.4s, v30.4s, v28.4s\n" + "zip2 v17.4s, v29.4s, v27.4s\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip2 v20.4s, v20.4s, v19.4s\n" + "zip2 v16.4s, v26.4s, v18.4s\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "sadalp v2.8h, v23.16b\n" + "sadalp v1.8h, v22.16b\n" + "str q25, [%x[out_ptr], #0x0]\n" + "add x20, x20, #0x1\n" + "zip1 v19.4s, v21.4s, v17.4s\n" + "zip1 v18.4s, v20.4s, v16.4s\n" + "str q24, [%x[out_ptr], #0x10]\n" + "sadalp v2.8h, v19.16b\n" + "sadalp v1.8h, v18.16b\n" + "str q23, [%x[out_ptr], #0x20]\n" + "zip2 v17.4s, v21.4s, v17.4s\n" + "zip2 v16.4s, v20.4s, v16.4s\n" + "str q22, [%x[out_ptr], #0x30]\n" + "str q19, [%x[out_ptr], #0x40]\n" + "sadalp v2.8h, v17.16b\n" + "sadalp v1.8h, v16.16b\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 14f\n" + "tbz %x[width], #3, 9f\n" + "ldr d29, [x28], #0x8\n" + "ldr d28, [x27], #0x8\n" + "ldr d27, [x26], #0x8\n" + "ldr d26, [x25], #0x8\n" + "ldr d25, [x24], #0x8\n" + "ldr d24, [x23], #0x8\n" + "ldr d23, [x22], #0x8\n" + "ldr d22, [x21], #0x8\n" + "tbz %x[width], #2, 7f\n" + "ld1 { v29.s }[2], [x28], #0x4\n" + "ld1 { v28.s }[2], [x27], #0x4\n" + "ld1 { v27.s }[2], [x26], #0x4\n" + "ld1 { v26.s }[2], [x25], #0x4\n" + "ld1 { v25.s }[2], [x24], #0x4\n" + "ld1 { v24.s }[2], [x23], #0x4\n" + "ld1 { v23.s }[2], [x22], #0x4\n" + "ld1 { v22.s }[2], [x21], #0x4\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v29.h }[6], [x28], #0x2\n" + "ld1 { v28.h }[6], [x27], #0x2\n" + "mov x20, #0x4\n" + "ld1 { v27.h }[6], [x26], #0x2\n" + "ld1 { v26.h }[6], [x25], #0x2\n" + "ld1 { v25.h }[6], [x24], #0x2\n" + "ld1 { v24.h }[6], [x23], #0x2\n" + "ld1 { v23.h }[6], [x22], #0x2\n" + "ld1 { v22.h }[6], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[14], [x28]\n" + "ld1 { v28.b }[14], [x27]\n" + "ld1 { v27.b }[14], [x26]\n" + "ld1 { v26.b }[14], [x25]\n" + "ld1 { v25.b }[14], [x24]\n" + "ld1 { v24.b }[14], [x23]\n" + "ld1 { v23.b }[14], [x22]\n" + "ld1 { v22.b }[14], [x21]\n" + "b 13f\n" + "6:" // odd_loads_1_12 + "mov x20, #0x3\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[12], [x28]\n" + "ld1 { v28.b }[12], [x27]\n" + "mov x20, #0x4\n" + "ld1 { v27.b }[12], [x26]\n" + "ld1 { v26.b }[12], [x25]\n" + "ld1 { v25.b }[12], [x24]\n" + "ld1 { v24.b }[12], [x23]\n" + "ld1 { v23.b }[12], [x22]\n" + "ld1 { v22.b }[12], [x21]\n" + "b 13f\n" + "7:" // odd_loads_2_8 + "tbz %x[width], #1, 8f\n" + "ld1 { v29.h }[4], [x28], #0x2\n" + "ld1 { v28.h }[4], [x27], #0x2\n" + "mov x20, #0x3\n" + "ld1 { v27.h }[4], [x26], #0x2\n" + "ld1 { v26.h }[4], [x25], #0x2\n" + "ld1 { v25.h }[4], [x24], #0x2\n" + "ld1 { v24.h }[4], [x23], #0x2\n" + "ld1 { v23.h }[4], [x22], #0x2\n" + "ld1 { v22.h }[4], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[10], [x28]\n" + "ld1 { v28.b }[10], [x27]\n" + "ld1 { v27.b }[10], [x26]\n" + "ld1 { v26.b }[10], [x25]\n" + "ld1 { v25.b }[10], [x24]\n" + "ld1 { v24.b }[10], [x23]\n" + "ld1 { v23.b }[10], [x22]\n" + "ld1 { v22.b }[10], [x21]\n" + "b 13f\n" + "8:" // odd_loads_1_8 + "mov x20, #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[8], [x28]\n" + "ld1 { v28.b }[8], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v27.b }[8], [x26]\n" + "ld1 { v26.b }[8], [x25]\n" + "ld1 { v25.b }[8], [x24]\n" + "ld1 { v24.b }[8], [x23]\n" + "ld1 { v23.b }[8], [x22]\n" + "ld1 { v22.b }[8], [x21]\n" + "b 13f\n" + "9:" // odd_loads_4_0 + "tbz %x[width], #2, 11f\n" + "ldr s29, [x28], #0x4\n" + "ldr s28, [x27], #0x4\n" + "ldr s27, [x26], #0x4\n" + "ldr s26, [x25], #0x4\n" + "ldr s25, [x24], #0x4\n" + "ldr s24, [x23], #0x4\n" + "ldr s23, [x22], #0x4\n" + "ldr s22, [x21], #0x4\n" + "tbz %x[width], #1, 10f\n" + "ld1 { v29.h }[2], [x28], #0x2\n" + "ld1 { v28.h }[2], [x27], #0x2\n" + "mov x20, #0x2\n" + "ld1 { v27.h }[2], [x26], #0x2\n" + "ld1 { v26.h }[2], [x25], #0x2\n" + "ld1 { v25.h }[2], [x24], #0x2\n" + "ld1 { v24.h }[2], [x23], #0x2\n" + "ld1 { v23.h }[2], [x22], #0x2\n" + "ld1 { v22.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[6], [x28]\n" + "ld1 { v28.b }[6], [x27]\n" + "ld1 { v27.b }[6], [x26]\n" + "ld1 { v26.b }[6], [x25]\n" + "ld1 { v25.b }[6], [x24]\n" + "ld1 { v24.b }[6], [x23]\n" + "ld1 { v23.b }[6], [x22]\n" + "ld1 { v22.b }[6], [x21]\n" + "b 13f\n" + "10:" // odd_loads_1_4 + "mov x20, #0x1\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[4], [x28]\n" + "ld1 { v28.b }[4], [x27]\n" + "mov x20, #0x2\n" + "ld1 { v27.b }[4], [x26]\n" + "ld1 { v26.b }[4], [x25]\n" + "ld1 { v25.b }[4], [x24]\n" + "ld1 { v24.b }[4], [x23]\n" + "ld1 { v23.b }[4], [x22]\n" + "ld1 { v22.b }[4], [x21]\n" + "b 13f\n" + "11:" // odd_loads_2_0 + "tbz %x[width], #1, 12f\n" + "ldr h29, [x28], #0x2\n" + "ldr h28, [x27], #0x2\n" + "mov x20, #0x1\n" + "ldr h27, [x26], #0x2\n" + "ldr h26, [x25], #0x2\n" + "ldr h25, [x24], #0x2\n" + "ldr h24, [x23], #0x2\n" + "ldr h23, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[2], [x28]\n" + "ld1 { v28.b }[2], [x27]\n" + "ld1 { v27.b }[2], [x26]\n" + "ld1 { v26.b }[2], [x25]\n" + "ld1 { v25.b }[2], [x24]\n" + "ld1 { v24.b }[2], [x23]\n" + "ld1 { v23.b }[2], [x22]\n" + "ld1 { v22.b }[2], [x21]\n" + "b 13f\n" + "12:" // odd_loads_1_0 + "ldr b29, [x28, #0x0]\n" + "ldr b28, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b27, [x26, #0x0]\n" + "ldr b26, [x25, #0x0]\n" + "ldr b25, [x24, #0x0]\n" + "ldr b24, [x23, #0x0]\n" + "ldr b23, [x22, #0x0]\n" + "ldr b22, [x21, #0x0]\n" + "13:" // Odd load end + "zip1 v21.4s, v29.4s, v27.4s\n" + "zip1 v20.4s, v28.4s, v26.4s\n" + "subs x20, x20, #0x1\n" + "zip1 v19.4s, v25.4s, v23.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v21.4s, v20.4s\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "sadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "sadalp v1.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 14f\n" + "zip2 v17.4s, v21.4s, v20.4s\n" + "zip2 v16.4s, v19.4s, v18.4s\n" + "subs x20, x20, #0x1\n" + "str q17, [%x[out_ptr], #0x0]\n" + "sadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "sadalp v1.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 14f\n" + "zip2 v21.4s, v29.4s, v27.4s\n" + "zip2 v20.4s, v28.4s, v26.4s\n" + "subs x20, x20, #0x1\n" + "zip2 v19.4s, v25.4s, v23.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v21.4s, v20.4s\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "sadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "sadalp v1.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 14f\n" + "zip2 v17.4s, v21.4s, v20.4s\n" + "zip2 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "sadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "sadalp v1.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "14:" // Odds skip + "sadalp v0.4s, v2.8h\n" + "sadalp v31.4s, v1.8h\n" + "str q0, [%x[out_ptr], #0x0]\n" + "str q31, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp new file mode 100644 index 0000000000..17eb7d5556 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp @@ -0,0 +1,370 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 4, VLType::None, true>( + uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "mov x20, #0x0\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "movi v2.8h, #0x0\n" + "movi v1.8h, #0x0\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "movi v0.4s, #0x0\n" + "movi v31.4s, #0x0\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x28, x28, %x[row_offset]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x20\n" + "ld1 { v0.4s }, [%x[out_ptr]]\n" + "ldr q31, [%x[out_ptr], #0x10]\n" + "2:" // first_pass + "cmp %x[width], #0x10\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x20, #0x1e\n" + "ble 4f\n" + "uadalp v0.4s, v2.8h\n" + "movi v2.8h, #0x0\n" + "mov x20, #0x0\n" + "uadalp v31.4s, v1.8h\n" + "movi v1.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr q30, [x28], #0x10\n" + "ldr q29, [x27], #0x10\n" + "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" + "ldr q28, [x26], #0x10\n" + "ldr q27, [x25], #0x10\n" + "zip1 v22.4s, v30.4s, v28.4s\n" + "zip1 v21.4s, v29.4s, v27.4s\n" + "ldr q20, [x24], #0x10\n" + "ldr q26, [x23], #0x10\n" + "zip1 v25.4s, v22.4s, v21.4s\n" + "uadalp v2.8h, v25.16b\n" + "ldr q19, [x22], #0x10\n" + "ldr q18, [x21], #0x10\n" + "zip1 v17.4s, v20.4s, v19.4s\n" + "zip1 v16.4s, v26.4s, v18.4s\n" + "zip1 v24.4s, v17.4s, v16.4s\n" + "uadalp v1.8h, v24.16b\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "zip2 v23.4s, v22.4s, v21.4s\n" + "zip2 v22.4s, v17.4s, v16.4s\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "zip2 v21.4s, v30.4s, v28.4s\n" + "zip2 v17.4s, v29.4s, v27.4s\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "zip2 v20.4s, v20.4s, v19.4s\n" + "zip2 v16.4s, v26.4s, v18.4s\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "uadalp v2.8h, v23.16b\n" + "uadalp v1.8h, v22.16b\n" + "str q25, [%x[out_ptr], #0x0]\n" + "add x20, x20, #0x1\n" + "zip1 v19.4s, v21.4s, v17.4s\n" + "zip1 v18.4s, v20.4s, v16.4s\n" + "str q24, [%x[out_ptr], #0x10]\n" + "uadalp v2.8h, v19.16b\n" + "uadalp v1.8h, v18.16b\n" + "str q23, [%x[out_ptr], #0x20]\n" + "zip2 v17.4s, v21.4s, v17.4s\n" + "zip2 v16.4s, v20.4s, v16.4s\n" + "str q22, [%x[out_ptr], #0x30]\n" + "str q19, [%x[out_ptr], #0x40]\n" + "uadalp v2.8h, v17.16b\n" + "uadalp v1.8h, v16.16b\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 14f\n" + "tbz %x[width], #3, 9f\n" + "ldr d29, [x28], #0x8\n" + "ldr d28, [x27], #0x8\n" + "ldr d27, [x26], #0x8\n" + "ldr d26, [x25], #0x8\n" + "ldr d25, [x24], #0x8\n" + "ldr d24, [x23], #0x8\n" + "ldr d23, [x22], #0x8\n" + "ldr d22, [x21], #0x8\n" + "tbz %x[width], #2, 7f\n" + "ld1 { v29.s }[2], [x28], #0x4\n" + "ld1 { v28.s }[2], [x27], #0x4\n" + "ld1 { v27.s }[2], [x26], #0x4\n" + "ld1 { v26.s }[2], [x25], #0x4\n" + "ld1 { v25.s }[2], [x24], #0x4\n" + "ld1 { v24.s }[2], [x23], #0x4\n" + "ld1 { v23.s }[2], [x22], #0x4\n" + "ld1 { v22.s }[2], [x21], #0x4\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v29.h }[6], [x28], #0x2\n" + "ld1 { v28.h }[6], [x27], #0x2\n" + "mov x20, #0x4\n" + "ld1 { v27.h }[6], [x26], #0x2\n" + "ld1 { v26.h }[6], [x25], #0x2\n" + "ld1 { v25.h }[6], [x24], #0x2\n" + "ld1 { v24.h }[6], [x23], #0x2\n" + "ld1 { v23.h }[6], [x22], #0x2\n" + "ld1 { v22.h }[6], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[14], [x28]\n" + "ld1 { v28.b }[14], [x27]\n" + "ld1 { v27.b }[14], [x26]\n" + "ld1 { v26.b }[14], [x25]\n" + "ld1 { v25.b }[14], [x24]\n" + "ld1 { v24.b }[14], [x23]\n" + "ld1 { v23.b }[14], [x22]\n" + "ld1 { v22.b }[14], [x21]\n" + "b 13f\n" + "6:" // odd_loads_1_12 + "mov x20, #0x3\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[12], [x28]\n" + "ld1 { v28.b }[12], [x27]\n" + "mov x20, #0x4\n" + "ld1 { v27.b }[12], [x26]\n" + "ld1 { v26.b }[12], [x25]\n" + "ld1 { v25.b }[12], [x24]\n" + "ld1 { v24.b }[12], [x23]\n" + "ld1 { v23.b }[12], [x22]\n" + "ld1 { v22.b }[12], [x21]\n" + "b 13f\n" + "7:" // odd_loads_2_8 + "tbz %x[width], #1, 8f\n" + "ld1 { v29.h }[4], [x28], #0x2\n" + "ld1 { v28.h }[4], [x27], #0x2\n" + "mov x20, #0x3\n" + "ld1 { v27.h }[4], [x26], #0x2\n" + "ld1 { v26.h }[4], [x25], #0x2\n" + "ld1 { v25.h }[4], [x24], #0x2\n" + "ld1 { v24.h }[4], [x23], #0x2\n" + "ld1 { v23.h }[4], [x22], #0x2\n" + "ld1 { v22.h }[4], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[10], [x28]\n" + "ld1 { v28.b }[10], [x27]\n" + "ld1 { v27.b }[10], [x26]\n" + "ld1 { v26.b }[10], [x25]\n" + "ld1 { v25.b }[10], [x24]\n" + "ld1 { v24.b }[10], [x23]\n" + "ld1 { v23.b }[10], [x22]\n" + "ld1 { v22.b }[10], [x21]\n" + "b 13f\n" + "8:" // odd_loads_1_8 + "mov x20, #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[8], [x28]\n" + "ld1 { v28.b }[8], [x27]\n" + "mov x20, #0x3\n" + "ld1 { v27.b }[8], [x26]\n" + "ld1 { v26.b }[8], [x25]\n" + "ld1 { v25.b }[8], [x24]\n" + "ld1 { v24.b }[8], [x23]\n" + "ld1 { v23.b }[8], [x22]\n" + "ld1 { v22.b }[8], [x21]\n" + "b 13f\n" + "9:" // odd_loads_4_0 + "tbz %x[width], #2, 11f\n" + "ldr s29, [x28], #0x4\n" + "ldr s28, [x27], #0x4\n" + "ldr s27, [x26], #0x4\n" + "ldr s26, [x25], #0x4\n" + "ldr s25, [x24], #0x4\n" + "ldr s24, [x23], #0x4\n" + "ldr s23, [x22], #0x4\n" + "ldr s22, [x21], #0x4\n" + "tbz %x[width], #1, 10f\n" + "ld1 { v29.h }[2], [x28], #0x2\n" + "ld1 { v28.h }[2], [x27], #0x2\n" + "mov x20, #0x2\n" + "ld1 { v27.h }[2], [x26], #0x2\n" + "ld1 { v26.h }[2], [x25], #0x2\n" + "ld1 { v25.h }[2], [x24], #0x2\n" + "ld1 { v24.h }[2], [x23], #0x2\n" + "ld1 { v23.h }[2], [x22], #0x2\n" + "ld1 { v22.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[6], [x28]\n" + "ld1 { v28.b }[6], [x27]\n" + "ld1 { v27.b }[6], [x26]\n" + "ld1 { v26.b }[6], [x25]\n" + "ld1 { v25.b }[6], [x24]\n" + "ld1 { v24.b }[6], [x23]\n" + "ld1 { v23.b }[6], [x22]\n" + "ld1 { v22.b }[6], [x21]\n" + "b 13f\n" + "10:" // odd_loads_1_4 + "mov x20, #0x1\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[4], [x28]\n" + "ld1 { v28.b }[4], [x27]\n" + "mov x20, #0x2\n" + "ld1 { v27.b }[4], [x26]\n" + "ld1 { v26.b }[4], [x25]\n" + "ld1 { v25.b }[4], [x24]\n" + "ld1 { v24.b }[4], [x23]\n" + "ld1 { v23.b }[4], [x22]\n" + "ld1 { v22.b }[4], [x21]\n" + "b 13f\n" + "11:" // odd_loads_2_0 + "tbz %x[width], #1, 12f\n" + "ldr h29, [x28], #0x2\n" + "ldr h28, [x27], #0x2\n" + "mov x20, #0x1\n" + "ldr h27, [x26], #0x2\n" + "ldr h26, [x25], #0x2\n" + "ldr h25, [x24], #0x2\n" + "ldr h24, [x23], #0x2\n" + "ldr h23, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v29.b }[2], [x28]\n" + "ld1 { v28.b }[2], [x27]\n" + "ld1 { v27.b }[2], [x26]\n" + "ld1 { v26.b }[2], [x25]\n" + "ld1 { v25.b }[2], [x24]\n" + "ld1 { v24.b }[2], [x23]\n" + "ld1 { v23.b }[2], [x22]\n" + "ld1 { v22.b }[2], [x21]\n" + "b 13f\n" + "12:" // odd_loads_1_0 + "ldr b29, [x28, #0x0]\n" + "ldr b28, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b27, [x26, #0x0]\n" + "ldr b26, [x25, #0x0]\n" + "ldr b25, [x24, #0x0]\n" + "ldr b24, [x23, #0x0]\n" + "ldr b23, [x22, #0x0]\n" + "ldr b22, [x21, #0x0]\n" + "13:" // Odd load end + "zip1 v21.4s, v29.4s, v27.4s\n" + "zip1 v20.4s, v28.4s, v26.4s\n" + "subs x20, x20, #0x1\n" + "zip1 v19.4s, v25.4s, v23.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v21.4s, v20.4s\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "uadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "uadalp v1.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 14f\n" + "zip2 v17.4s, v21.4s, v20.4s\n" + "zip2 v16.4s, v19.4s, v18.4s\n" + "subs x20, x20, #0x1\n" + "str q17, [%x[out_ptr], #0x0]\n" + "uadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "uadalp v1.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 14f\n" + "zip2 v21.4s, v29.4s, v27.4s\n" + "zip2 v20.4s, v28.4s, v26.4s\n" + "subs x20, x20, #0x1\n" + "zip2 v19.4s, v25.4s, v23.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v21.4s, v20.4s\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "uadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "uadalp v1.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "beq 14f\n" + "zip2 v17.4s, v21.4s, v20.4s\n" + "zip2 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "uadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "uadalp v1.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + "14:" // Odds skip + "uadalp v0.4s, v2.8h\n" + "uadalp v31.4s, v1.8h\n" + "str q0, [%x[out_ptr], #0x0]\n" + "str q31, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp new file mode 100644 index 0000000000..7b445ef3d4 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp @@ -0,0 +1,318 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 8, VLType::None, false>( + int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "add x28, x28, %x[row_offset]\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "cmp %x[width], #0x10\n" + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "blt 3f\n" + "2:" // Main loop head + "ldr q20, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" + "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" + "ldr q25, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" + "zip1 v16.2d, v20.2d, v19.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" + "ldr q23, [x24], #0x10\n" + "ldr q22, [x23], #0x10\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip2 v21.2d, v20.2d, v19.2d\n" + "ldr q20, [x22], #0x10\n" + "ldr q19, [x21], #0x10\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v20.2d, v19.2d\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "str q18, [%x[out_ptr], #0x10]\n" + "zip2 v18.2d, v25.2d, v24.2d\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "str q17, [%x[out_ptr], #0x20]\n" + "zip2 v17.2d, v23.2d, v22.2d\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "zip2 v16.2d, v20.2d, v19.2d\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "str q21, [%x[out_ptr], #0x40]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 2b\n" + "3:" // Main loop skip + "cbz %x[width], 12f\n" + "tbz %x[width], #3, 7f\n" + "ldr d25, [x28], #0x8\n" + "ldr d24, [x27], #0x8\n" + "ldr d23, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d19, [x22], #0x8\n" + "ldr d18, [x21], #0x8\n" + "tbz %x[width], #2, 5f\n" + "ld1 { v25.s }[2], [x28], #0x4\n" + "ld1 { v24.s }[2], [x27], #0x4\n" + "ld1 { v23.s }[2], [x26], #0x4\n" + "ld1 { v22.s }[2], [x25], #0x4\n" + "ld1 { v21.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v19.s }[2], [x22], #0x4\n" + "ld1 { v18.s }[2], [x21], #0x4\n" + "tbz %x[width], #1, 4f\n" + "ld1 { v25.h }[6], [x28], #0x2\n" + "ld1 { v24.h }[6], [x27], #0x2\n" + "mov x20, #0x2\n" + "ld1 { v23.h }[6], [x26], #0x2\n" + "ld1 { v22.h }[6], [x25], #0x2\n" + "ld1 { v21.h }[6], [x24], #0x2\n" + "ld1 { v20.h }[6], [x23], #0x2\n" + "ld1 { v19.h }[6], [x22], #0x2\n" + "ld1 { v18.h }[6], [x21], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v25.b }[14], [x28]\n" + "ld1 { v24.b }[14], [x27]\n" + "ld1 { v23.b }[14], [x26]\n" + "ld1 { v22.b }[14], [x25]\n" + "ld1 { v21.b }[14], [x24]\n" + "ld1 { v20.b }[14], [x23]\n" + "ld1 { v19.b }[14], [x22]\n" + "ld1 { v18.b }[14], [x21]\n" + "b 11f\n" + "4:" // odd_loads_1_12 + "mov x20, #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v25.b }[12], [x28]\n" + "ld1 { v24.b }[12], [x27]\n" + "ld1 { v23.b }[12], [x26]\n" + "ld1 { v22.b }[12], [x25]\n" + "ld1 { v21.b }[12], [x24]\n" + "ld1 { v20.b }[12], [x23]\n" + "ld1 { v19.b }[12], [x22]\n" + "ld1 { v18.b }[12], [x21]\n" + "b 11f\n" + "5:" // odd_loads_2_8 + "tbz %x[width], #1, 6f\n" + "ld1 { v25.h }[4], [x28], #0x2\n" + "ld1 { v24.h }[4], [x27], #0x2\n" + "mov x20, #0x2\n" + "ld1 { v23.h }[4], [x26], #0x2\n" + "ld1 { v22.h }[4], [x25], #0x2\n" + "ld1 { v21.h }[4], [x24], #0x2\n" + "ld1 { v20.h }[4], [x23], #0x2\n" + "ld1 { v19.h }[4], [x22], #0x2\n" + "ld1 { v18.h }[4], [x21], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v25.b }[10], [x28]\n" + "ld1 { v24.b }[10], [x27]\n" + "ld1 { v23.b }[10], [x26]\n" + "ld1 { v22.b }[10], [x25]\n" + "ld1 { v21.b }[10], [x24]\n" + "ld1 { v20.b }[10], [x23]\n" + "ld1 { v19.b }[10], [x22]\n" + "ld1 { v18.b }[10], [x21]\n" + "b 11f\n" + "6:" // odd_loads_1_8 + "mov x20, #0x1\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v25.b }[8], [x28]\n" + "ld1 { v24.b }[8], [x27]\n" + "mov x20, #0x2\n" + "ld1 { v23.b }[8], [x26]\n" + "ld1 { v22.b }[8], [x25]\n" + "ld1 { v21.b }[8], [x24]\n" + "ld1 { v20.b }[8], [x23]\n" + "ld1 { v19.b }[8], [x22]\n" + "ld1 { v18.b }[8], [x21]\n" + "b 11f\n" + "7:" // odd_loads_4_0 + "tbz %x[width], #2, 9f\n" + "ldr s25, [x28], #0x4\n" + "ldr s24, [x27], #0x4\n" + "ldr s23, [x26], #0x4\n" + "ldr s22, [x25], #0x4\n" + "ldr s21, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s19, [x22], #0x4\n" + "ldr s18, [x21], #0x4\n" + "tbz %x[width], #1, 8f\n" + "ld1 { v25.h }[2], [x28], #0x2\n" + "ld1 { v24.h }[2], [x27], #0x2\n" + "mov x20, #0x1\n" + "ld1 { v23.h }[2], [x26], #0x2\n" + "ld1 { v22.h }[2], [x25], #0x2\n" + "ld1 { v21.h }[2], [x24], #0x2\n" + "ld1 { v20.h }[2], [x23], #0x2\n" + "ld1 { v19.h }[2], [x22], #0x2\n" + "ld1 { v18.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v25.b }[6], [x28]\n" + "ld1 { v24.b }[6], [x27]\n" + "ld1 { v23.b }[6], [x26]\n" + "ld1 { v22.b }[6], [x25]\n" + "ld1 { v21.b }[6], [x24]\n" + "ld1 { v20.b }[6], [x23]\n" + "ld1 { v19.b }[6], [x22]\n" + "ld1 { v18.b }[6], [x21]\n" + "b 11f\n" + "8:" // odd_loads_1_4 + "mov x20, #0x1\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v25.b }[4], [x28]\n" + "ld1 { v24.b }[4], [x27]\n" + "ld1 { v23.b }[4], [x26]\n" + "ld1 { v22.b }[4], [x25]\n" + "ld1 { v21.b }[4], [x24]\n" + "ld1 { v20.b }[4], [x23]\n" + "ld1 { v19.b }[4], [x22]\n" + "ld1 { v18.b }[4], [x21]\n" + "b 11f\n" + "9:" // odd_loads_2_0 + "tbz %x[width], #1, 10f\n" + "ldr h25, [x28], #0x2\n" + "ldr h24, [x27], #0x2\n" + "mov x20, #0x1\n" + "ldr h23, [x26], #0x2\n" + "ldr h22, [x25], #0x2\n" + "ldr h21, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h19, [x22], #0x2\n" + "ldr h18, [x21], #0x2\n" + "tbz %x[width], #0, 11f\n" + "ld1 { v25.b }[2], [x28]\n" + "ld1 { v24.b }[2], [x27]\n" + "ld1 { v23.b }[2], [x26]\n" + "ld1 { v22.b }[2], [x25]\n" + "ld1 { v21.b }[2], [x24]\n" + "ld1 { v20.b }[2], [x23]\n" + "ld1 { v19.b }[2], [x22]\n" + "ld1 { v18.b }[2], [x21]\n" + "b 11f\n" + "10:" // odd_loads_1_0 + "ldr b25, [x28, #0x0]\n" + "ldr b24, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b23, [x26, #0x0]\n" + "ldr b22, [x25, #0x0]\n" + "ldr b21, [x24, #0x0]\n" + "ldr b20, [x23, #0x0]\n" + "ldr b19, [x22, #0x0]\n" + "ldr b18, [x21, #0x0]\n" + "11:" // Odd load end + "subs x20, x20, #0x1\n" + "zip1 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip1 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.2d, v21.2d, v20.2d\n" + "zip1 v16.2d, v19.2d, v18.2d\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "beq 12f\n" + "zip2 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v17.2d, v21.2d, v20.2d\n" + "zip2 v16.2d, v19.2d, v18.2d\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "12:" // Odds skip + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + +template<> +void interleave_block<8, 8, VLType::None, false>( + uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height, + size_t row_offset, bool +) +{ + int8_t * &out_cast = reinterpret_cast<int8_t * &>(out_ptr); + const int8_t * const * in_cast = reinterpret_cast<const int8_t * const *>(in); + + interleave_block<8, 8, VLType::None, false>(out_cast, in_cast, width, height, row_offset, false); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp new file mode 100644 index 0000000000..a2288e8299 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 8, VLType::None, true>( + int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "mov x20, #0x0\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "movi v5.8h, #0x0\n" + "movi v4.8h, #0x0\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "movi v3.8h, #0x0\n" + "movi v2.8h, #0x0\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "movi v1.4s, #0x0\n" + "movi v0.4s, #0x0\n" + "movi v31.4s, #0x0\n" + "movi v30.4s, #0x0\n" + "add x28, x28, %x[row_offset]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "movi v29.4s, #0x0\n" + "movi v28.4s, #0x0\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x20\n" + "ld1 { v29.4s }, [%x[out_ptr]]\n" + "ldr q28, [%x[out_ptr], #0x10]\n" + "2:" // first_pass + "cmp %x[width], #0x10\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x20, #0x3e\n" + "ble 4f\n" + "sadalp v1.4s, v5.8h\n" + "movi v5.8h, #0x0\n" + "mov x20, #0x0\n" + "sadalp v0.4s, v4.8h\n" + "movi v4.8h, #0x0\n" + "sadalp v31.4s, v3.8h\n" + "movi v3.8h, #0x0\n" + "sadalp v30.4s, v2.8h\n" + "movi v2.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr q27, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" + "zip1 v26.2d, v27.2d, v19.2d\n" + "sadalp v5.8h, v26.16b\n" + "ldr q25, [x26], #0x10\n" + "ldr q18, [x25], #0x10\n" + "zip1 v24.2d, v25.2d, v18.2d\n" + "sadalp v4.8h, v24.16b\n" + "ldr q23, [x24], #0x10\n" + "ldr q17, [x23], #0x10\n" + "zip1 v22.2d, v23.2d, v17.2d\n" + "sadalp v3.8h, v22.16b\n" + "ldr q21, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v20.2d, v21.2d, v16.2d\n" + "sadalp v2.8h, v20.16b\n" + "zip2 v19.2d, v27.2d, v19.2d\n" + "zip2 v18.2d, v25.2d, v18.2d\n" + "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" + "zip2 v17.2d, v23.2d, v17.2d\n" + "zip2 v16.2d, v21.2d, v16.2d\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "str q26, [%x[out_ptr], #0x0]\n" + "sadalp v5.8h, v19.16b\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "str q24, [%x[out_ptr], #0x10]\n" + "sadalp v4.8h, v18.16b\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "str q22, [%x[out_ptr], #0x20]\n" + "sadalp v3.8h, v17.16b\n" + "str q20, [%x[out_ptr], #0x30]\n" + "sadalp v2.8h, v16.16b\n" + "add x20, x20, #0x1\n" + "str q19, [%x[out_ptr], #0x40]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 14f\n" + "tbz %x[width], #3, 9f\n" + "ldr d27, [x28], #0x8\n" + "ldr d26, [x27], #0x8\n" + "ldr d25, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "ldr d23, [x24], #0x8\n" + "ldr d22, [x23], #0x8\n" + "ldr d21, [x22], #0x8\n" + "ldr d20, [x21], #0x8\n" + "tbz %x[width], #2, 7f\n" + "ld1 { v27.s }[2], [x28], #0x4\n" + "ld1 { v26.s }[2], [x27], #0x4\n" + "ld1 { v25.s }[2], [x26], #0x4\n" + "ld1 { v24.s }[2], [x25], #0x4\n" + "ld1 { v23.s }[2], [x24], #0x4\n" + "ld1 { v22.s }[2], [x23], #0x4\n" + "ld1 { v21.s }[2], [x22], #0x4\n" + "ld1 { v20.s }[2], [x21], #0x4\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v27.h }[6], [x28], #0x2\n" + "ld1 { v26.h }[6], [x27], #0x2\n" + "mov x20, #0x2\n" + "ld1 { v25.h }[6], [x26], #0x2\n" + "ld1 { v24.h }[6], [x25], #0x2\n" + "ld1 { v23.h }[6], [x24], #0x2\n" + "ld1 { v22.h }[6], [x23], #0x2\n" + "ld1 { v21.h }[6], [x22], #0x2\n" + "ld1 { v20.h }[6], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[14], [x28]\n" + "ld1 { v26.b }[14], [x27]\n" + "ld1 { v25.b }[14], [x26]\n" + "ld1 { v24.b }[14], [x25]\n" + "ld1 { v23.b }[14], [x24]\n" + "ld1 { v22.b }[14], [x23]\n" + "ld1 { v21.b }[14], [x22]\n" + "ld1 { v20.b }[14], [x21]\n" + "b 13f\n" + "6:" // odd_loads_1_12 + "mov x20, #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[12], [x28]\n" + "ld1 { v26.b }[12], [x27]\n" + "ld1 { v25.b }[12], [x26]\n" + "ld1 { v24.b }[12], [x25]\n" + "ld1 { v23.b }[12], [x24]\n" + "ld1 { v22.b }[12], [x23]\n" + "ld1 { v21.b }[12], [x22]\n" + "ld1 { v20.b }[12], [x21]\n" + "b 13f\n" + "7:" // odd_loads_2_8 + "tbz %x[width], #1, 8f\n" + "ld1 { v27.h }[4], [x28], #0x2\n" + "ld1 { v26.h }[4], [x27], #0x2\n" + "mov x20, #0x2\n" + "ld1 { v25.h }[4], [x26], #0x2\n" + "ld1 { v24.h }[4], [x25], #0x2\n" + "ld1 { v23.h }[4], [x24], #0x2\n" + "ld1 { v22.h }[4], [x23], #0x2\n" + "ld1 { v21.h }[4], [x22], #0x2\n" + "ld1 { v20.h }[4], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[10], [x28]\n" + "ld1 { v26.b }[10], [x27]\n" + "ld1 { v25.b }[10], [x26]\n" + "ld1 { v24.b }[10], [x25]\n" + "ld1 { v23.b }[10], [x24]\n" + "ld1 { v22.b }[10], [x23]\n" + "ld1 { v21.b }[10], [x22]\n" + "ld1 { v20.b }[10], [x21]\n" + "b 13f\n" + "8:" // odd_loads_1_8 + "mov x20, #0x1\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[8], [x28]\n" + "ld1 { v26.b }[8], [x27]\n" + "mov x20, #0x2\n" + "ld1 { v25.b }[8], [x26]\n" + "ld1 { v24.b }[8], [x25]\n" + "ld1 { v23.b }[8], [x24]\n" + "ld1 { v22.b }[8], [x23]\n" + "ld1 { v21.b }[8], [x22]\n" + "ld1 { v20.b }[8], [x21]\n" + "b 13f\n" + "9:" // odd_loads_4_0 + "tbz %x[width], #2, 11f\n" + "ldr s27, [x28], #0x4\n" + "ldr s26, [x27], #0x4\n" + "ldr s25, [x26], #0x4\n" + "ldr s24, [x25], #0x4\n" + "ldr s23, [x24], #0x4\n" + "ldr s22, [x23], #0x4\n" + "ldr s21, [x22], #0x4\n" + "ldr s20, [x21], #0x4\n" + "tbz %x[width], #1, 10f\n" + "ld1 { v27.h }[2], [x28], #0x2\n" + "ld1 { v26.h }[2], [x27], #0x2\n" + "mov x20, #0x1\n" + "ld1 { v25.h }[2], [x26], #0x2\n" + "ld1 { v24.h }[2], [x25], #0x2\n" + "ld1 { v23.h }[2], [x24], #0x2\n" + "ld1 { v22.h }[2], [x23], #0x2\n" + "ld1 { v21.h }[2], [x22], #0x2\n" + "ld1 { v20.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[6], [x28]\n" + "ld1 { v26.b }[6], [x27]\n" + "ld1 { v25.b }[6], [x26]\n" + "ld1 { v24.b }[6], [x25]\n" + "ld1 { v23.b }[6], [x24]\n" + "ld1 { v22.b }[6], [x23]\n" + "ld1 { v21.b }[6], [x22]\n" + "ld1 { v20.b }[6], [x21]\n" + "b 13f\n" + "10:" // odd_loads_1_4 + "mov x20, #0x1\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[4], [x28]\n" + "ld1 { v26.b }[4], [x27]\n" + "ld1 { v25.b }[4], [x26]\n" + "ld1 { v24.b }[4], [x25]\n" + "ld1 { v23.b }[4], [x24]\n" + "ld1 { v22.b }[4], [x23]\n" + "ld1 { v21.b }[4], [x22]\n" + "ld1 { v20.b }[4], [x21]\n" + "b 13f\n" + "11:" // odd_loads_2_0 + "tbz %x[width], #1, 12f\n" + "ldr h27, [x28], #0x2\n" + "ldr h26, [x27], #0x2\n" + "mov x20, #0x1\n" + "ldr h25, [x26], #0x2\n" + "ldr h24, [x25], #0x2\n" + "ldr h23, [x24], #0x2\n" + "ldr h22, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h20, [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[2], [x28]\n" + "ld1 { v26.b }[2], [x27]\n" + "ld1 { v25.b }[2], [x26]\n" + "ld1 { v24.b }[2], [x25]\n" + "ld1 { v23.b }[2], [x24]\n" + "ld1 { v22.b }[2], [x23]\n" + "ld1 { v21.b }[2], [x22]\n" + "ld1 { v20.b }[2], [x21]\n" + "b 13f\n" + "12:" // odd_loads_1_0 + "ldr b27, [x28, #0x0]\n" + "ldr b26, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b25, [x26, #0x0]\n" + "ldr b24, [x25, #0x0]\n" + "ldr b23, [x24, #0x0]\n" + "ldr b22, [x23, #0x0]\n" + "ldr b21, [x22, #0x0]\n" + "ldr b20, [x21, #0x0]\n" + "13:" // Odd load end + "zip1 v19.2d, v27.2d, v26.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" + "subs x20, x20, #0x1\n" + "str q19, [%x[out_ptr], #0x0]\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip1 v16.2d, v21.2d, v20.2d\n" + "str q18, [%x[out_ptr], #0x10]\n" + "sadalp v5.8h, v19.16b\n" + "sadalp v4.8h, v18.16b\n" + "str q17, [%x[out_ptr], #0x20]\n" + "sadalp v3.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x30]\n" + "sadalp v2.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "beq 14f\n" + "zip2 v19.2d, v27.2d, v26.2d\n" + "zip2 v18.2d, v25.2d, v24.2d\n" + "str q19, [%x[out_ptr], #0x0]\n" + "zip2 v17.2d, v23.2d, v22.2d\n" + "zip2 v16.2d, v21.2d, v20.2d\n" + "str q18, [%x[out_ptr], #0x10]\n" + "sadalp v5.8h, v19.16b\n" + "sadalp v4.8h, v18.16b\n" + "str q17, [%x[out_ptr], #0x20]\n" + "sadalp v3.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x30]\n" + "sadalp v2.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "14:" // Odds skip + "sadalp v1.4s, v5.8h\n" + "sadalp v0.4s, v4.8h\n" + "sadalp v31.4s, v3.8h\n" + "sadalp v30.4s, v2.8h\n" + "addp v1.4s, v1.4s, v0.4s\n" + "addp v16.4s, v31.4s, v30.4s\n" + "add v1.4s, v1.4s, v29.4s\n" + "add v16.4s, v16.4s, v28.4s\n" + "str q1, [%x[out_ptr], #0x0]\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp new file mode 100644 index 0000000000..56d34a8a64 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2019-2021, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef __aarch64__ + +template<> +void interleave_block<8, 8, VLType::None, true>( + uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height, + size_t row_offset, bool first +) +{ + __asm__ __volatile__( + "ldr x28, [%x[in], #0x0]\n" + "ldr x27, [%x[in], #0x8]\n" + "cmp %x[height], #0x8\n" + "mov x20, #0x0\n" + "ldr x26, [%x[in], #0x10]\n" + "ldr x25, [%x[in], #0x18]\n" + "movi v5.8h, #0x0\n" + "movi v4.8h, #0x0\n" + "ldr x24, [%x[in], #0x20]\n" + "ldr x23, [%x[in], #0x28]\n" + "movi v3.8h, #0x0\n" + "movi v2.8h, #0x0\n" + "ldr x22, [%x[in], #0x30]\n" + "ldr x21, [%x[in], #0x38]\n" + "movi v1.4s, #0x0\n" + "movi v0.4s, #0x0\n" + "movi v31.4s, #0x0\n" + "movi v30.4s, #0x0\n" + "add x28, x28, %x[row_offset]\n" + "add x27, x27, %x[row_offset]\n" + "add x26, x26, %x[row_offset]\n" + "add x25, x25, %x[row_offset]\n" + "add x24, x24, %x[row_offset]\n" + "add x23, x23, %x[row_offset]\n" + "add x22, x22, %x[row_offset]\n" + "add x21, x21, %x[row_offset]\n" + "beq 1f\n" + "cmp %x[height], #0x2\n" + "csel x27, x27, x28, GE\n" + "csel x26, x26, x28, GT\n" + "cmp %x[height], #0x4\n" + "csel x25, x25, x28, GE\n" + "csel x24, x24, x28, GT\n" + "cmp %x[height], #0x6\n" + "mov x21, x28\n" + "csel x23, x23, x28, GE\n" + "csel x22, x22, x28, GT\n" + "1:" // no_pointer_adj + "prfm pldl1keep, [x28, #0x0]\n" + "prfm pldl1keep, [x27, #0x0]\n" + "movi v29.4s, #0x0\n" + "movi v28.4s, #0x0\n" + "prfm pldl1keep, [x26, #0x0]\n" + "prfm pldl1keep, [x25, #0x0]\n" + "prfm pldl1keep, [x24, #0x0]\n" + "prfm pldl1keep, [x23, #0x0]\n" + "prfm pldl1keep, [x22, #0x0]\n" + "prfm pldl1keep, [x21, #0x0]\n" + "prfm pldl1keep, [x28, #0x40]\n" + "prfm pldl1keep, [x27, #0x40]\n" + "prfm pldl1keep, [x26, #0x40]\n" + "prfm pldl1keep, [x25, #0x40]\n" + "prfm pldl1keep, [x24, #0x40]\n" + "prfm pldl1keep, [x23, #0x40]\n" + "prfm pldl1keep, [x22, #0x40]\n" + "prfm pldl1keep, [x21, #0x40]\n" + "cbnz %w[first], 2f\n" + "sub %x[out_ptr], %x[out_ptr], #0x20\n" + "ld1 { v29.4s }, [%x[out_ptr]]\n" + "ldr q28, [%x[out_ptr], #0x10]\n" + "2:" // first_pass + "cmp %x[width], #0x10\n" + "blt 5f\n" + "3:" // Main loop head + "cmp x20, #0x3e\n" + "ble 4f\n" + "uadalp v1.4s, v5.8h\n" + "movi v5.8h, #0x0\n" + "mov x20, #0x0\n" + "uadalp v0.4s, v4.8h\n" + "movi v4.8h, #0x0\n" + "uadalp v31.4s, v3.8h\n" + "movi v3.8h, #0x0\n" + "uadalp v30.4s, v2.8h\n" + "movi v2.8h, #0x0\n" + "4:" // no_accumulate_16 + "ldr q27, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" + "zip1 v26.2d, v27.2d, v19.2d\n" + "uadalp v5.8h, v26.16b\n" + "ldr q25, [x26], #0x10\n" + "ldr q18, [x25], #0x10\n" + "zip1 v24.2d, v25.2d, v18.2d\n" + "uadalp v4.8h, v24.16b\n" + "ldr q23, [x24], #0x10\n" + "ldr q17, [x23], #0x10\n" + "zip1 v22.2d, v23.2d, v17.2d\n" + "uadalp v3.8h, v22.16b\n" + "ldr q21, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v20.2d, v21.2d, v16.2d\n" + "uadalp v2.8h, v20.16b\n" + "zip2 v19.2d, v27.2d, v19.2d\n" + "zip2 v18.2d, v25.2d, v18.2d\n" + "subs %x[width], %x[width], #0x10\n" + "cmp %x[width], #0x10\n" + "zip2 v17.2d, v23.2d, v17.2d\n" + "zip2 v16.2d, v21.2d, v16.2d\n" + "prfm pldl1keep, [x28, #0x70]\n" + "prfm pldl1keep, [x27, #0x70]\n" + "prfm pldl1keep, [x26, #0x70]\n" + "prfm pldl1keep, [x25, #0x70]\n" + "str q26, [%x[out_ptr], #0x0]\n" + "uadalp v5.8h, v19.16b\n" + "prfm pldl1keep, [x24, #0x70]\n" + "prfm pldl1keep, [x23, #0x70]\n" + "str q24, [%x[out_ptr], #0x10]\n" + "uadalp v4.8h, v18.16b\n" + "prfm pldl1keep, [x22, #0x70]\n" + "prfm pldl1keep, [x21, #0x70]\n" + "str q22, [%x[out_ptr], #0x20]\n" + "uadalp v3.8h, v17.16b\n" + "str q20, [%x[out_ptr], #0x30]\n" + "uadalp v2.8h, v16.16b\n" + "add x20, x20, #0x1\n" + "str q19, [%x[out_ptr], #0x40]\n" + "str q18, [%x[out_ptr], #0x50]\n" + "str q17, [%x[out_ptr], #0x60]\n" + "str q16, [%x[out_ptr], #0x70]\n" + "add %x[out_ptr], %x[out_ptr], #0x80\n" + "bge 3b\n" + "5:" // Main loop skip + "cbz %x[width], 14f\n" + "tbz %x[width], #3, 9f\n" + "ldr d27, [x28], #0x8\n" + "ldr d26, [x27], #0x8\n" + "ldr d25, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "ldr d23, [x24], #0x8\n" + "ldr d22, [x23], #0x8\n" + "ldr d21, [x22], #0x8\n" + "ldr d20, [x21], #0x8\n" + "tbz %x[width], #2, 7f\n" + "ld1 { v27.s }[2], [x28], #0x4\n" + "ld1 { v26.s }[2], [x27], #0x4\n" + "ld1 { v25.s }[2], [x26], #0x4\n" + "ld1 { v24.s }[2], [x25], #0x4\n" + "ld1 { v23.s }[2], [x24], #0x4\n" + "ld1 { v22.s }[2], [x23], #0x4\n" + "ld1 { v21.s }[2], [x22], #0x4\n" + "ld1 { v20.s }[2], [x21], #0x4\n" + "tbz %x[width], #1, 6f\n" + "ld1 { v27.h }[6], [x28], #0x2\n" + "ld1 { v26.h }[6], [x27], #0x2\n" + "mov x20, #0x2\n" + "ld1 { v25.h }[6], [x26], #0x2\n" + "ld1 { v24.h }[6], [x25], #0x2\n" + "ld1 { v23.h }[6], [x24], #0x2\n" + "ld1 { v22.h }[6], [x23], #0x2\n" + "ld1 { v21.h }[6], [x22], #0x2\n" + "ld1 { v20.h }[6], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[14], [x28]\n" + "ld1 { v26.b }[14], [x27]\n" + "ld1 { v25.b }[14], [x26]\n" + "ld1 { v24.b }[14], [x25]\n" + "ld1 { v23.b }[14], [x24]\n" + "ld1 { v22.b }[14], [x23]\n" + "ld1 { v21.b }[14], [x22]\n" + "ld1 { v20.b }[14], [x21]\n" + "b 13f\n" + "6:" // odd_loads_1_12 + "mov x20, #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[12], [x28]\n" + "ld1 { v26.b }[12], [x27]\n" + "ld1 { v25.b }[12], [x26]\n" + "ld1 { v24.b }[12], [x25]\n" + "ld1 { v23.b }[12], [x24]\n" + "ld1 { v22.b }[12], [x23]\n" + "ld1 { v21.b }[12], [x22]\n" + "ld1 { v20.b }[12], [x21]\n" + "b 13f\n" + "7:" // odd_loads_2_8 + "tbz %x[width], #1, 8f\n" + "ld1 { v27.h }[4], [x28], #0x2\n" + "ld1 { v26.h }[4], [x27], #0x2\n" + "mov x20, #0x2\n" + "ld1 { v25.h }[4], [x26], #0x2\n" + "ld1 { v24.h }[4], [x25], #0x2\n" + "ld1 { v23.h }[4], [x24], #0x2\n" + "ld1 { v22.h }[4], [x23], #0x2\n" + "ld1 { v21.h }[4], [x22], #0x2\n" + "ld1 { v20.h }[4], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[10], [x28]\n" + "ld1 { v26.b }[10], [x27]\n" + "ld1 { v25.b }[10], [x26]\n" + "ld1 { v24.b }[10], [x25]\n" + "ld1 { v23.b }[10], [x24]\n" + "ld1 { v22.b }[10], [x23]\n" + "ld1 { v21.b }[10], [x22]\n" + "ld1 { v20.b }[10], [x21]\n" + "b 13f\n" + "8:" // odd_loads_1_8 + "mov x20, #0x1\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[8], [x28]\n" + "ld1 { v26.b }[8], [x27]\n" + "mov x20, #0x2\n" + "ld1 { v25.b }[8], [x26]\n" + "ld1 { v24.b }[8], [x25]\n" + "ld1 { v23.b }[8], [x24]\n" + "ld1 { v22.b }[8], [x23]\n" + "ld1 { v21.b }[8], [x22]\n" + "ld1 { v20.b }[8], [x21]\n" + "b 13f\n" + "9:" // odd_loads_4_0 + "tbz %x[width], #2, 11f\n" + "ldr s27, [x28], #0x4\n" + "ldr s26, [x27], #0x4\n" + "ldr s25, [x26], #0x4\n" + "ldr s24, [x25], #0x4\n" + "ldr s23, [x24], #0x4\n" + "ldr s22, [x23], #0x4\n" + "ldr s21, [x22], #0x4\n" + "ldr s20, [x21], #0x4\n" + "tbz %x[width], #1, 10f\n" + "ld1 { v27.h }[2], [x28], #0x2\n" + "ld1 { v26.h }[2], [x27], #0x2\n" + "mov x20, #0x1\n" + "ld1 { v25.h }[2], [x26], #0x2\n" + "ld1 { v24.h }[2], [x25], #0x2\n" + "ld1 { v23.h }[2], [x24], #0x2\n" + "ld1 { v22.h }[2], [x23], #0x2\n" + "ld1 { v21.h }[2], [x22], #0x2\n" + "ld1 { v20.h }[2], [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[6], [x28]\n" + "ld1 { v26.b }[6], [x27]\n" + "ld1 { v25.b }[6], [x26]\n" + "ld1 { v24.b }[6], [x25]\n" + "ld1 { v23.b }[6], [x24]\n" + "ld1 { v22.b }[6], [x23]\n" + "ld1 { v21.b }[6], [x22]\n" + "ld1 { v20.b }[6], [x21]\n" + "b 13f\n" + "10:" // odd_loads_1_4 + "mov x20, #0x1\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[4], [x28]\n" + "ld1 { v26.b }[4], [x27]\n" + "ld1 { v25.b }[4], [x26]\n" + "ld1 { v24.b }[4], [x25]\n" + "ld1 { v23.b }[4], [x24]\n" + "ld1 { v22.b }[4], [x23]\n" + "ld1 { v21.b }[4], [x22]\n" + "ld1 { v20.b }[4], [x21]\n" + "b 13f\n" + "11:" // odd_loads_2_0 + "tbz %x[width], #1, 12f\n" + "ldr h27, [x28], #0x2\n" + "ldr h26, [x27], #0x2\n" + "mov x20, #0x1\n" + "ldr h25, [x26], #0x2\n" + "ldr h24, [x25], #0x2\n" + "ldr h23, [x24], #0x2\n" + "ldr h22, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h20, [x21], #0x2\n" + "tbz %x[width], #0, 13f\n" + "ld1 { v27.b }[2], [x28]\n" + "ld1 { v26.b }[2], [x27]\n" + "ld1 { v25.b }[2], [x26]\n" + "ld1 { v24.b }[2], [x25]\n" + "ld1 { v23.b }[2], [x24]\n" + "ld1 { v22.b }[2], [x23]\n" + "ld1 { v21.b }[2], [x22]\n" + "ld1 { v20.b }[2], [x21]\n" + "b 13f\n" + "12:" // odd_loads_1_0 + "ldr b27, [x28, #0x0]\n" + "ldr b26, [x27, #0x0]\n" + "mov x20, #0x1\n" + "ldr b25, [x26, #0x0]\n" + "ldr b24, [x25, #0x0]\n" + "ldr b23, [x24, #0x0]\n" + "ldr b22, [x23, #0x0]\n" + "ldr b21, [x22, #0x0]\n" + "ldr b20, [x21, #0x0]\n" + "13:" // Odd load end + "zip1 v19.2d, v27.2d, v26.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" + "subs x20, x20, #0x1\n" + "str q19, [%x[out_ptr], #0x0]\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip1 v16.2d, v21.2d, v20.2d\n" + "str q18, [%x[out_ptr], #0x10]\n" + "uadalp v5.8h, v19.16b\n" + "uadalp v4.8h, v18.16b\n" + "str q17, [%x[out_ptr], #0x20]\n" + "uadalp v3.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x30]\n" + "uadalp v2.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "beq 14f\n" + "zip2 v19.2d, v27.2d, v26.2d\n" + "zip2 v18.2d, v25.2d, v24.2d\n" + "str q19, [%x[out_ptr], #0x0]\n" + "zip2 v17.2d, v23.2d, v22.2d\n" + "zip2 v16.2d, v21.2d, v20.2d\n" + "str q18, [%x[out_ptr], #0x10]\n" + "uadalp v5.8h, v19.16b\n" + "uadalp v4.8h, v18.16b\n" + "str q17, [%x[out_ptr], #0x20]\n" + "uadalp v3.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x30]\n" + "uadalp v2.8h, v16.16b\n" + "add %x[out_ptr], %x[out_ptr], #0x40\n" + "14:" // Odds skip + "uadalp v1.4s, v5.8h\n" + "uadalp v0.4s, v4.8h\n" + "uadalp v31.4s, v3.8h\n" + "uadalp v30.4s, v2.8h\n" + "addp v1.4s, v1.4s, v0.4s\n" + "addp v16.4s, v31.4s, v30.4s\n" + "add v1.4s, v1.4s, v29.4s\n" + "add v16.4s, v16.4s, v28.4s\n" + "str q1, [%x[out_ptr], #0x0]\n" + "str q16, [%x[out_ptr], #0x10]\n" + "add %x[out_ptr], %x[out_ptr], #0x20\n" + : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + ); +} + + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list-sve.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list-sve.hpp new file mode 100644 index 0000000000..57f26ac135 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list-sve.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "sme_interleave1VL_bf16_bf16.hpp" +#include "sme_interleave1VL_block2_bf16_bf16.hpp" +#include "sme_interleave1VL_block2_fp16_fp16.hpp" +#include "sme_interleave1VL_block4_s8_s8.hpp" +#include "sme_interleave1VL_block4_u8_u8.hpp" +#include "sme_interleave1VL_block4_s8_s8_summing.hpp" +#include "sme_interleave1VL_block4_u8_u8_summing.hpp" +#include "sme_interleave1VL_fp16_fp16.hpp" +#include "sme_interleave1VL_fp32_fp32.hpp" +#include "sme_interleave2VL_block2_bf16_bf16.hpp" +#include "sme_interleave2VL_block2_fp16_fp16.hpp" +#include "sme_interleave2VL_block4_s8_s8.hpp" +#include "sme_interleave2VL_block4_s8_s8_summing.hpp" +#include "sme_interleave2VL_block4_u8_u8.hpp" +#include "sme_interleave2VL_block4_u8_u8_summing.hpp" +#include "sme_interleave2VL_fp16_fp16.hpp" +#include "sme_interleave2VL_bf16_bf16.hpp" +#include "sme_interleave2VL_fp32_fp32.hpp" +#include "sme_interleave4VL_block2_bf16_bf16.hpp" +#include "sme_interleave4VL_block2_fp16_fp16.hpp" +#include "sme_interleave4VL_block4_s8_s8.hpp" +#include "sme_interleave4VL_block4_u8_u8.hpp" +#include "sme_interleave4VL_block4_s8_s8_summing.hpp" +#include "sme_interleave4VL_block4_u8_u8_summing.hpp" +#include "sme_interleave4VL_fp32_fp32.hpp" + +#include "sme2_interleave1VL_block2_fp32_bf16.hpp" +#include "sme2_interleave2VL_block2_fp32_bf16.hpp" +#include "sme2_interleave4VL_block2_fp32_bf16.hpp" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list.hpp new file mode 100644 index 0000000000..b13d32c324 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "a32_interleave6_block1_fp32_fp32.hpp" +#include "a64_interleave4_block16_s8_s8.hpp" +#include "a64_interleave4_block16_s8_s8_summing.hpp" +#include "a64_interleave4_block16_u8_u8_summing.hpp" +#include "a64_interleave8_block1_bf16_fp32.hpp" +#include "a64_interleave8_block1_fp16_fp16.hpp" +#include "a64_interleave8_block1_fp16_fp32.hpp" +#include "a64_interleave8_block1_fp32_fp32.hpp" +#include "a64_interleave8_block1_s16_s16.hpp" +#include "a64_interleave8_block1_s16_s16_summing.hpp" +#include "a64_interleave8_block1_s8_s16.hpp" +#include "a64_interleave8_block1_s8_s16_summing.hpp" +#include "a64_interleave8_block1_u16_u16_summing.hpp" +#include "a64_interleave8_block1_u8_u16.hpp" +#include "a64_interleave8_block1_u8_u16_summing.hpp" +#include "a64_interleave8_block2_bf16_bf16.hpp" +#include "a64_interleave8_block2_fp32_fp32.hpp" +#include "a64_interleave8_block4_bf16_bf16.hpp" +#include "a64_interleave8_block4_fp32_bf16.hpp" +#include "a64_interleave8_block4_s8_s8.hpp" +#include "a64_interleave8_block4_s8_s8_summing.hpp" +#include "a64_interleave8_block4_u8_u8_summing.hpp" +#include "a64_interleave8_block8_s8_s8.hpp" +#include "a64_interleave8_block8_s8_s8_summing.hpp" +#include "a64_interleave8_block8_u8_u8_summing.hpp" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp new file mode 100644 index 0000000000..a5f4754d3d --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME2) + +template <> +void interleave_block<1, 2, VLType::SME, false>( + bfloat16 * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x22, ALL, MUL #2\n" + "sub x28, %x[width], #0x1\n" + "cntw x21, ALL, MUL #2\n" + "sub x20, x22, #0x1\n" + "whilelt p10.s, XZR, %x[height]\n" + "add x28, x28, x21\n" + "ands x27, %x[width], x20\n" + "udiv x28, x28, x21\n" + "csel x27, x27, x22, NE\n" + "mov x26, #0x0\n" + "and x25, x28, #0x1\n" + "sub x28, x28, #0x1\n" + "add x27, x27, #0x1\n" + "mov x20, %x[width]\n" + "ptrue p0.b\n" + "mov x24, %x[outptr_raw]\n" + "mov x23, %x[row_offset]\n" + "cntw x22\n" + "lsr x28, x28, #0x1\n" + "lsr x27, x27, #0x1\n" + "mov x12, #0x0\n" + ".inst 0x25b44751 // whilelt pn9.s, x26, x20, VLx2\n" + "mov x21, %x[in]\n" + "1:" // Width loop: Preamble: Loop + "ldr x20, [x21], #0x8\n" + ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" + ".inst 0xa0174286 // ld1w { z6.s-z7.s }, pn8.s/Z, [x20, x23, LSL #2]\n" + ".inst 0xc160e0c6 // bfcvt z6.h, { z6.s-z7.s }\n" + ".inst 0xc08000c0 // mova za0h.s[x12], p0/M, z6.s\n" + "add x12, x12, #0x1\n" + "cmp x12, x22\n" + "blt 1b\n" + "incw x23, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" + "cbz x28, 5f\n" + "2:" // Width loop + "mov x20, %x[width]\n" + "mov x12, #0x0\n" + ".inst 0x25b44751 // whilelt pn9.s, x26, x20, VLx2\n" + "mov x21, %x[in]\n" + "3:" // Width loop: Odd: Loop + "ldr x20, [x21], #0x8\n" + ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" + ".inst 0xa017429e // ld1w { z30.s-z31.s }, pn8.s/Z, [x20, x23, LSL #2]\n" + ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" + ".inst 0xc08003c8 // mova za2h.s[x12], p0/M, z30.s\n" + ".inst 0xc082800f // mova z15.s, p0/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x22\n" + "st1w { z15.s }, p0, [x24]\n" + "addvl x24, x24, #1\n" + "blt 3b\n" + "incw x26, ALL, MUL #2\n" + "mov x20, %x[width]\n" + "incw x23, ALL, MUL #2\n" + "mov x12, #0x0\n" + ".inst 0x25b44751 // whilelt pn9.s, x26, x20, VLx2\n" + "mov x21, %x[in]\n" + "4:" // Width loop: Even: Loop + "ldr x20, [x21], #0x8\n" + ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" + ".inst 0xa0174298 // ld1w { z24.s-z25.s }, pn8.s/Z, [x20, x23, LSL #2]\n" + ".inst 0xc160e318 // bfcvt z24.h, { z24.s-z25.s }\n" + ".inst 0xc0800300 // mova za0h.s[x12], p0/M, z24.s\n" + ".inst 0xc0828110 // mova z16.s, p0/M, za2v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x22\n" + "st1w { z16.s }, p0, [x24]\n" + "addvl x24, x24, #1\n" + "blt 4b\n" + "subs x28, x28, #0x1\n" + "incw x23, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" + "bgt 2b\n" + "5:" // Width loop: Tails + "cbnz x25, 8f\n" + "mov x20, %x[width]\n" + "mov x12, #0x0\n" + ".inst 0x25b44751 // whilelt pn9.s, x26, x20, VLx2\n" + "mov x21, %x[in]\n" + "6:" // Width loop: Tails: Even: Odd: Loop + "ldr x20, [x21], #0x8\n" + ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" + ".inst 0xa017428e // ld1w { z14.s-z15.s }, pn8.s/Z, [x20, x23, LSL #2]\n" + ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" + ".inst 0xc08001c8 // mova za2h.s[x12], p0/M, z14.s\n" + ".inst 0xc0828010 // mova z16.s, p0/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x22\n" + "st1w { z16.s }, p0, [x24]\n" + "addvl x24, x24, #1\n" + "blt 6b\n" + "mov x12, #0x0\n" + "7:" // Width loop: Tails: Even: Even: Loop + ".inst 0xc0828110 // mova z16.s, p0/M, za2v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + "st1w { z16.s }, p0, [x24]\n" + "addvl x24, x24, #1\n" + "blt 7b\n" + "b 10f\n" + "8:" // Width loop: Tails: Odd + "mov x12, #0x0\n" + "9:" // Width loop: Tails: Odd: Loop + ".inst 0xc0828010 // mova z16.s, p0/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + "st1w { z16.s }, p0, [x24]\n" + "addvl x24, x24, #1\n" + "blt 9b\n" + "10:" // End + "mov %x[outptr_raw], x24\n" + ".inst 0xd503467f // SMSTOP\n" + : [outptr_raw] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp new file mode 100644 index 0000000000..c1d0ac5bc7 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME2) + +template <> +void interleave_block<2, 2, VLType::SME, false>( + bfloat16 * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x22, ALL, MUL #2\n" + "cntw x9\n" + "sub x28, %x[width], #0x1\n" + "cntw x21, ALL, MUL #2\n" + "sub x20, x22, #0x1\n" + ".inst 0x25207815 // ptrue pn13.b\n" + "whilelt p12.s, XZR, %x[height]\n" + "whilelt p11.s, x9, %x[height]\n" + "add x28, x28, x21\n" + "ands x27, %x[width], x20\n" + "udiv x28, x28, x21\n" + "csel x27, x27, x22, NE\n" + "mov x26, #0x0\n" + "and x25, x28, #0x1\n" + "sub x28, x28, #0x1\n" + "add x27, x27, #0x1\n" + "mov x20, %x[width]\n" + "mov x24, %x[in]\n" + "ptrue p0.b\n" + "mov x23, %x[outptr_raw]\n" + "mov x22, %x[row_offset]\n" + "lsr x28, x28, #0x1\n" + "lsr x27, x27, #0x1\n" + "mov x12, #0x0\n" + ".inst 0x25b44752 // whilelt pn10.s, x26, x20, VLx2\n" + "add x21, x24, x9, LSL #3\n" + "1:" // Width loop: Preamble: Loop + "ldr x20, [x24], #0x8\n" + ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" + ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" + ".inst 0xa0164698 // ld1w { z24.s-z25.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa0164296 // ld1w { z22.s-z23.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e318 // bfcvt z24.h, { z24.s-z25.s }\n" + ".inst 0xc160e2d6 // bfcvt z22.h, { z22.s-z23.s }\n" + ".inst 0xc0800300 // mova za0h.s[x12], p0/M, z24.s\n" + ".inst 0xc08002c4 // mova za1h.s[x12], p0/M, z22.s\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "blt 1b\n" + "incw x22, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" + "cbz x28, 5f\n" + "2:" // Width loop + "mov x20, %x[width]\n" + "mov x24, %x[in]\n" + "mov x12, #0x0\n" + ".inst 0x25b44752 // whilelt pn10.s, x26, x20, VLx2\n" + "add x21, x24, x9, LSL #3\n" + "3:" // Width loop: Odd: Loop + "ldr x20, [x24], #0x8\n" + ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" + ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" + ".inst 0xa0164696 // ld1w { z22.s-z23.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016428a // ld1w { z10.s-z11.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e2d6 // bfcvt z22.h, { z22.s-z23.s }\n" + ".inst 0xc160e14a // bfcvt z10.h, { z10.s-z11.s }\n" + ".inst 0xc08002c8 // mova za2h.s[x12], p0/M, z22.s\n" + ".inst 0xc080014c // mova za3h.s[x12], p0/M, z10.s\n" + ".inst 0xc0828008 // mova z8.s, p0/M, za0v.s[x12]\n" + ".inst 0xc0828089 // mova z9.s, p0/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + ".inst 0xa06056e8 // st1w { z8.s-z9.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" + "blt 3b\n" + "incw x26, ALL, MUL #2\n" + "mov x20, %x[width]\n" + "mov x24, %x[in]\n" + "incw x22, ALL, MUL #2\n" + "mov x12, #0x0\n" + ".inst 0x25b44752 // whilelt pn10.s, x26, x20, VLx2\n" + "add x21, x24, x9, LSL #3\n" + "4:" // Width loop: Even: Loop + "ldr x20, [x24], #0x8\n" + ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" + ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" + ".inst 0xa016469a // ld1w { z26.s-z27.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016429e // ld1w { z30.s-z31.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e35a // bfcvt z26.h, { z26.s-z27.s }\n" + ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" + ".inst 0xc0800340 // mova za0h.s[x12], p0/M, z26.s\n" + ".inst 0xc08003c4 // mova za1h.s[x12], p0/M, z30.s\n" + ".inst 0xc0828106 // mova z6.s, p0/M, za2v.s[x12]\n" + ".inst 0xc082818e // mova z14.s, p0/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + ".inst 0xa16056e6 // st1w { z6.s, z14.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" + "blt 4b\n" + "subs x28, x28, #0x1\n" + "incw x22, ALL, MUL #2\n" + "incw x26, ALL, MUL #2\n" + "bgt 2b\n" + "5:" // Width loop: Tails + "cbnz x25, 8f\n" + "mov x20, %x[width]\n" + "mov x24, %x[in]\n" + "mov x12, #0x0\n" + ".inst 0x25b44752 // whilelt pn10.s, x26, x20, VLx2\n" + "add x21, x24, x9, LSL #3\n" + "6:" // Width loop: Tails: Even: Odd: Loop + "ldr x20, [x24], #0x8\n" + ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" + ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" + ".inst 0xa016468c // ld1w { z12.s-z13.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016428e // ld1w { z14.s-z15.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" + ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" + ".inst 0xc0800188 // mova za2h.s[x12], p0/M, z12.s\n" + ".inst 0xc08001cc // mova za3h.s[x12], p0/M, z14.s\n" + ".inst 0xc0828007 // mova z7.s, p0/M, za0v.s[x12]\n" + ".inst 0xc082808f // mova z15.s, p0/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + ".inst 0xa16056e7 // st1w { z7.s, z15.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" + "blt 6b\n" + "mov x12, #0x0\n" + "7:" // Width loop: Tails: Even: Even: Loop + ".inst 0xc082810e // mova z14.s, p0/M, za2v.s[x12]\n" + ".inst 0xc082818f // mova z15.s, p0/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + ".inst 0xa06056ee // st1w { z14.s-z15.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" + "blt 7b\n" + "b 10f\n" + "8:" // Width loop: Tails: Odd + "mov x12, #0x0\n" + "9:" // Width loop: Tails: Odd: Loop + ".inst 0xc0828014 // mova z20.s, p0/M, za0v.s[x12]\n" + ".inst 0xc0828095 // mova z21.s, p0/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + ".inst 0xa06056f4 // st1w { z20.s-z21.s }, pn13.b, [x23]\n" + "addvl x23, x23, #2\n" + "blt 9b\n" + "10:" // End + "mov %x[outptr_raw], x23\n" + ".inst 0xd503467f // SMSTOP\n" + : [outptr_raw] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp new file mode 100644 index 0000000000..03575d7ff2 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME2) + +template <> +void interleave_block<4, 2, VLType::SME, false>( + bfloat16 * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x23, ALL, MUL #2\n" + "cntw x10\n" + "cntw x22, ALL, MUL #2\n" + "cntw x20, ALL, MUL #3\n" + "sub x21, x23, #0x1\n" + ".inst 0x25207817 // ptrue pn15.b\n" + "whilelt p1.s, XZR, %x[height]\n" + "whilelt p14.s, x10, %x[height]\n" + "whilelt p13.s, x22, %x[height]\n" + "whilelt p12.s, x20, %x[height]\n" + "sub x9, %x[width], #0x1\n" + "cntw x20, ALL, MUL #2\n" + "ands x28, %x[width], x21\n" + "mov x27, %x[in]\n" + "add x9, x9, x20\n" + "csel x28, x28, x23, NE\n" + "add x26, x27, x10, LSL #3\n" + "mov x25, #0x0\n" + "udiv x9, x9, x20\n" + "add x28, x28, #0x1\n" + "mov x20, %x[width]\n" + "add x24, x26, x10, LSL #3\n" + "ptrue p0.b\n" + "mov x23, %x[outptr_raw]\n" + "mov x22, %x[row_offset]\n" + "sub x9, x9, #0x1\n" + "lsr x28, x28, #0x1\n" + "mov x12, #0x0\n" + ".inst 0x25b44733 // whilelt pn11.s, x25, x20, VLx2\n" + "add x21, x24, x10, LSL #3\n" + "1:" // Width loop: Preamble: Loop + "ldr x20, [x27], #0x8\n" + ".inst 0x25306c28 // psel p8.s, p11.s/Z, p1.s[w12]\n" + ".inst 0x25306dca // psel p10.s, p11.s/Z, p14.s[w12]\n" + ".inst 0xa0164298 // ld1w { z24.s-z25.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0x25306da9 // psel p9.s, p11.s/Z, p13.s[w12]\n" + ".inst 0x25306d88 // psel p8.s, p11.s/Z, p12.s[w12]\n" + ".inst 0xa0164a82 // ld1w { z2.s-z3.s }, pn10.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + ".inst 0xa016468a // ld1w { z10.s-z11.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e318 // bfcvt z24.h, { z24.s-z25.s }\n" + ".inst 0xc160e042 // bfcvt z2.h, { z2.s-z3.s }\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016428c // ld1w { z12.s-z13.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e14a // bfcvt z10.h, { z10.s-z11.s }\n" + ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" + ".inst 0xc0800300 // mova za0h.s[x12], p0/M, z24.s\n" + ".inst 0xc0800044 // mova za1h.s[x12], p0/M, z2.s\n" + ".inst 0xc0800148 // mova za2h.s[x12], p0/M, z10.s\n" + ".inst 0xc080018c // mova za3h.s[x12], p0/M, z12.s\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 1b\n" + "incw x22, ALL, MUL #2\n" + "incw x25, ALL, MUL #2\n" + "cbz x9, 5f\n" + "2:" // Width loop + "mov x12, #0x0\n" + "3:" // Width loop: Store: Loop + ".inst 0xc0828011 // mova z17.s, p0/M, za0v.s[x12]\n" + ".inst 0xc0828095 // mova z21.s, p0/M, za1v.s[x12]\n" + ".inst 0xc0828119 // mova z25.s, p0/M, za2v.s[x12]\n" + ".inst 0xc082819d // mova z29.s, p0/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + ".inst 0xa160def1 // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x23]\n" + "addvl x23, x23, #4\n" + "blt 3b\n" + "mov x27, %x[in]\n" + "add x26, x27, x10, LSL #3\n" + "mov x20, %x[width]\n" + "add x24, x26, x10, LSL #3\n" + "mov x12, #0x0\n" + ".inst 0x25b44733 // whilelt pn11.s, x25, x20, VLx2\n" + "add x21, x24, x10, LSL #3\n" + "4:" // Width loop: Load: Loop + "ldr x20, [x27], #0x8\n" + ".inst 0x25306c28 // psel p8.s, p11.s/Z, p1.s[w12]\n" + ".inst 0x25306dca // psel p10.s, p11.s/Z, p14.s[w12]\n" + ".inst 0xa016428c // ld1w { z12.s-z13.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0x25306da9 // psel p9.s, p11.s/Z, p13.s[w12]\n" + ".inst 0x25306d88 // psel p8.s, p11.s/Z, p12.s[w12]\n" + ".inst 0xa0164a8e // ld1w { z14.s-z15.s }, pn10.s/Z, [x20, x22, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + ".inst 0xa0164692 // ld1w { z18.s-z19.s }, pn9.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" + ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" + "ldr x20, [x21], #0x8\n" + ".inst 0xa016429e // ld1w { z30.s-z31.s }, pn8.s/Z, [x20, x22, LSL #2]\n" + ".inst 0xc160e252 // bfcvt z18.h, { z18.s-z19.s }\n" + ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" + ".inst 0xc0800180 // mova za0h.s[x12], p0/M, z12.s\n" + ".inst 0xc08001c4 // mova za1h.s[x12], p0/M, z14.s\n" + ".inst 0xc0800248 // mova za2h.s[x12], p0/M, z18.s\n" + ".inst 0xc08003cc // mova za3h.s[x12], p0/M, z30.s\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 4b\n" + "subs x9, x9, #0x1\n" + "incw x22, ALL, MUL #2\n" + "incw x25, ALL, MUL #2\n" + "bgt 2b\n" + "5:" // Width loop: Tails + "mov x12, #0x0\n" + "6:" // Width loop: Tails: Loop + ".inst 0xc0828011 // mova z17.s, p0/M, za0v.s[x12]\n" + ".inst 0xc0828095 // mova z21.s, p0/M, za1v.s[x12]\n" + ".inst 0xc0828119 // mova z25.s, p0/M, za2v.s[x12]\n" + ".inst 0xc082819d // mova z29.s, p0/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x28\n" + ".inst 0xa160def1 // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x23]\n" + "addvl x23, x23, #4\n" + "blt 6b\n" + "7:" // End + "mov %x[outptr_raw], x23\n" + ".inst 0xd503467f // SMSTOP\n" + : [outptr_raw] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp new file mode 100644 index 0000000000..453778ae3f --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<1, 1, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov x21, %x[width]\n" + "inch x21\n" + "cnth x11\n" + "sub x21, x21, #0x1\n" + "udiv x21, x21, x11\n" // n_passes = ceildiv(width, VL<T>) + "mov x20, %x[width]\n" + "sub x10, x11, #0x1\n" + "sub x9, x21, #0x1\n" + "ands x10, x20, x10\n" + "sub x28, x11, #0x2\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "mov x27, #0x0\n" + "mov x26, %x[in]\n" + "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 + "ldr x25, [x26, #0x0]\n" + "and x24, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "csel x10, x10, x11, NE\n" + "ldr x23, [x26, #0x8]\n" + "ptrue p11.h\n" + "whilelt p10.h, XZR, x20\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "add x26, x26, #0x10\n" + "mov x12, #0x0\n" + "cbz x28, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0xe05602e1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x23, x22, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" + "mov x26, %x[in]\n" + ".inst 0xe05602e1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x23, x22, LSL #1]\n" + "ldr x25, [x26, #0x0]\n" + "inch x22\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "inch x27\n" + "cbz x9, 8f\n" + "mov x20, x9\n" + "3:" // K loop: Main loop + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + "cbz x28, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06b82a1 // st1h { za0v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + "whilelt p9.h, x27, %x[width]\n" + "inch x27\n" + "add x26, x26, #0x10\n" + ".inst 0xe06b82a1 // st1h { za0v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "addvl x21, x21, #2\n" + "inch x22\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + "cbz x28, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06b82a9 // st1h { za1v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + "whilelt p9.h, x27, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x26, x26, #0x10\n" + ".inst 0xe06b82a9 // st1h { za1v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "addvl x21, x21, #2\n" + "inch x27\n" + "inch x22\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x24, 11f\n" + "mov x26, %x[in]\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + "ldr x20, [x26, #0x0]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x22, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x11\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" + "blt 9b\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "blt 10b\n" + "whilelt p8.h, x27, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x21\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp new file mode 100644 index 0000000000..98bdcd2fa2 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<1, 2, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x22\n" + "mov x21, %x[width]\n" + "inch x21\n" + "mov x20, %x[width]\n" + "sub x11, x22, #0x1\n" + "sub x21, x21, #0x1\n" + "ands x11, x20, x11\n" + "cntw x10\n" + "udiv x21, x21, x22\n" // n_passes = ceildiv(width, VL<T>) + "csel x11, x11, x22, NE\n" + "sub x9, x21, #0x1\n" + "add x11, x11, #0x1\n" + "sub x28, x10, #0x2\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "mov x27, #0x0\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x24, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x23, [x26, #0x8]\n" + "lsr x11, x11, #0x1\n" + "ptrue p11.s\n" + "whilelt p10.h, XZR, x20\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "add x26, x26, #0x10\n" + "mov x12, #0x0\n" + "cbz x28, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0xe05602e2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" + "add x12, x12, #0x4\n" + "cmp x12, x28, LSL #1\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" + "mov x26, %x[in]\n" + ".inst 0xe05602e2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" + "ldr x25, [x26, #0x0]\n" + "inch x22\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "inch x27\n" + "cbz x9, 8f\n" + "mov x20, x9\n" + "3:" // K loop: Main loop + "whilelt p8.h, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x28, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" + ".inst 0xe0562321 // ld1h { za0h.h[x13, #1] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25796141 // psel p1.h, p8.h/Z, p10.h[w13, #3]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe05626e3 // ld1h { za0h.h[x13, #3] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0aa82a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x4\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" + ".inst 0xe0562321 // ld1h { za0h.h[x13, #1] }, p0/Z, [x25, x22, LSL #1]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25796141 // psel p1.h, p8.h/Z, p10.h[w13, #3]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe05626e3 // ld1h { za0h.h[x13, #3] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.h, x27, %x[width]\n" + "inch x27\n" + "add x26, x26, #0x10\n" + ".inst 0xe0aa82a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "addvl x21, x21, #2\n" + "inch x22\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x28, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25296140 // psel p0.h, p8.h/Z, p10.h[w13]\n" + ".inst 0xe0562320 // ld1h { za0h.h[x13] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25696141 // psel p1.h, p8.h/Z, p10.h[w13, #2]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe05626e2 // ld1h { za0h.h[x13, #2] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x4\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25296140 // psel p0.h, p8.h/Z, p10.h[w13]\n" + ".inst 0xe0562320 // ld1h { za0h.h[x13] }, p0/Z, [x25, x22, LSL #1]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25696141 // psel p1.h, p8.h/Z, p10.h[w13, #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe05626e2 // ld1h { za0h.h[x13, #2] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.h, x27, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x26, x26, #0x10\n" + ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "addvl x21, x21, #2\n" + "inch x27\n" + "inch x22\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x24, 11f\n" + "mov x26, %x[in]\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x20, [x26, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" + "cmp x12, x10\n" + ".inst 0xe0562281 // ld1h { za0h.h[x13, #1] }, p0/Z, [x20, x22, LSL #1]\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" + "add x13, x13, #0x2\n" + "blt 9b\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x11\n" + "addvl x21, x21, #1\n" + "add x20, x20, #0x2\n" + "blt 10b\n" + "whilelt p8.h, x27, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x11\n" + "addvl x21, x21, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x21\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_fp16_fp16.hpp new file mode 100644 index 0000000000..30c3e42aed --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_fp16_fp16.hpp @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 2, VLType::SME, false>( + __fp16 * &out, const __fp16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov x22, %x[width]\n" + "mov x21, %x[width]\n" + "cnth x20\n" + "inch x22\n" + "sub x11, x20, #0x1\n" + "sub x22, x22, #0x1\n" + "ands x11, x21, x11\n" + "cntw x10\n" + "udiv x22, x22, x20\n" // n_passes = ceildiv(width, VL<T>) + "csel x11, x11, x20, NE\n" + "sub x9, x22, #0x1\n" + "add x11, x11, #0x1\n" + "sub x28, x10, #0x2\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "mov x27, #0x0\n" + "mov x26, %x[in]\n" + "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x25, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x24, [x26, #0x0]\n" + "lsr x11, x11, #0x1\n" + "ptrue p11.s\n" + "ldr x23, [x26, #0x8]\n" + "whilelt p10.h, XZR, x20\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "add x26, x26, #0x10\n" + "mov x12, #0x0\n" + "cbz x28, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286143 // psel p3.h, p8.h/Z, p10.h[w12]\n" + ".inst 0x25686142 // psel p2.h, p8.h/Z, p10.h[w12, #2]\n" + ".inst 0xe0560f00 // ld1h { za0h.h[x12] }, p3/Z, [x24, x22, LSL #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe0560ae2 // ld1h { za0h.h[x12, #2] }, p2/Z, [x23, x22, LSL #1]\n" + "add x12, x12, #0x4\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "cmp x12, x28, LSL #1\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286141 // psel p1.h, p8.h/Z, p10.h[w12]\n" + ".inst 0x25686140 // psel p0.h, p8.h/Z, p10.h[w12, #2]\n" + "mov x26, %x[in]\n" + "inch x27\n" + ".inst 0xe0560700 // ld1h { za0h.h[x12] }, p1/Z, [x24, x22, LSL #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe05602e2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "inch x22\n" + "cbz x9, 8f\n" + "mov x20, x9\n" + "3:" // K loop: Main loop + "whilelt p8.h, x27, %x[width]\n" + "mov x15, #0x0\n" + "mov x14, #0x0\n" + "cbz x28, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x253b6143 // psel p3.h, p8.h/Z, p10.h[w15, #1]\n" + ".inst 0x257b6142 // psel p2.h, p8.h/Z, p10.h[w15, #3]\n" + ".inst 0x252a6d21 // psel p1.h, p11.h/Z, p9.h[w14]\n" + ".inst 0x253a6d20 // psel p0.h, p11.h/Z, p9.h[w14, #1]\n" + ".inst 0xe0566f01 // ld1h { za0h.h[x15, #1] }, p3/Z, [x24, x22, LSL #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe0566ae3 // ld1h { za0h.h[x15, #3] }, p2/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "add x15, x15, #0x4\n" + ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0aac2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "add x14, x14, #0x2\n" + "addvl x21, x21, #2\n" + "cmp x14, x28\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x253b6143 // psel p3.h, p8.h/Z, p10.h[w15, #1]\n" + ".inst 0x257b6142 // psel p2.h, p8.h/Z, p10.h[w15, #3]\n" + ".inst 0x252a6d21 // psel p1.h, p11.h/Z, p9.h[w14]\n" + ".inst 0x253a6d20 // psel p0.h, p11.h/Z, p9.h[w14, #1]\n" + "mov x26, %x[in]\n" + "whilelt p9.h, x27, %x[width]\n" + ".inst 0xe0566f01 // ld1h { za0h.h[x15, #1] }, p3/Z, [x24, x22, LSL #1]\n" + "ldr x24, [x26, #0x0]\n" + "inch x27\n" + "mov x13, #0x0\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + ".inst 0xe0566ae3 // ld1h { za0h.h[x15, #3] }, p2/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "inch x22\n" + ".inst 0xe0bfc6a0 // st1w { za0v.s[x14] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0aac2a1 // st1w { za0v.s[x14, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "addvl x21, x21, #2\n" + "cbz x28, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25296143 // psel p3.h, p8.h/Z, p10.h[w13]\n" + ".inst 0x25696142 // psel p2.h, p8.h/Z, p10.h[w13, #2]\n" + ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe0562f00 // ld1h { za0h.h[x13] }, p3/Z, [x24, x22, LSL #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe0562ae2 // ld1h { za0h.h[x13, #2] }, p2/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "add x13, x13, #0x4\n" + ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "add x12, x12, #0x2\n" + "addvl x21, x21, #2\n" + "cmp x12, x28\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25296143 // psel p3.h, p8.h/Z, p10.h[w13]\n" + ".inst 0x25696142 // psel p2.h, p8.h/Z, p10.h[w13, #2]\n" + ".inst 0x25286d21 // psel p1.h, p11.h/Z, p9.h[w12]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + "mov x26, %x[in]\n" + "whilelt p9.h, x27, %x[width]\n" + ".inst 0xe0562f00 // ld1h { za0h.h[x13] }, p3/Z, [x24, x22, LSL #1]\n" + "ldr x24, [x26, #0x0]\n" + "subs x20, x20, #0x1\n" + "inch x27\n" + ".inst 0xe0562ae2 // ld1h { za0h.h[x13, #2] }, p2/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "inch x22\n" + ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "addvl x21, x21, #2\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x25, 11f\n" + "mov x26, %x[in]\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25396143 // psel p3.h, p8.h/Z, p10.h[w13, #1]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x24, [x26, #0x0]\n" + "add x12, x12, #0x1\n" + "add x26, x26, #0x8\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + ".inst 0xe0562f01 // ld1h { za0h.h[x13, #1] }, p3/Z, [x24, x22, LSL #1]\n" + "add x13, x13, #0x2\n" + "blt 9b\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + "add x20, x20, #0x2\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "addvl x21, x21, #1\n" + "cmp x12, x11\n" + "blt 10b\n" + "whilelt p9.h, x27, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x11\n" + "addvl x21, x21, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x21\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp new file mode 100644 index 0000000000..4390bb7c7f --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<1, 4, VLType::SME, false>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" + "mov x23, %x[width]\n" + "incb x23\n" + "mov x20, %x[width]\n" + "sub x10, x21, #0x1\n" + "cntw x9\n" + "sub x23, x23, #0x1\n" + "ands x10, x20, x10\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x10, x10, x21, NE\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x21, x9, #0x1\n" + "sub x20, x23, #0x1\n" + "add x10, x10, #0x3\n" + "sub x28, x9, #0x2\n" + "whilelt p9.b, XZR, x22\n" + "whilelt p8.b, x21, x22\n" + "mov x27, #0x0\n" + "mov x26, %x[in]\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "ldr x25, [x26, #0x0]\n" + "and x24, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "lsr x10, x10, #0x2\n" + "ldr x23, [x26, #0x8]\n" + "ptrue p11.s\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" + "add x26, x26, #0x10\n" + "mov x12, #0x0\n" + "cbz x28, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x22]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0xe01602e4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x23, x22]\n" + "add x12, x12, #0x8\n" + "cmp x12, x28, LSL #2\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x22]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "mov x26, %x[in]\n" + ".inst 0xe01602e4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x23, x22]\n" + "ldr x25, [x26, #0x0]\n" + "incb x22\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "incb x27\n" + "cbz x20, 8f\n" + "mov x20, x20\n" + "3:" // K loop: Main loop + "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x28, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0a982a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x8\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x27, %x[width]\n" + "incb x27\n" + "add x26, x26, #0x10\n" + ".inst 0xe0a982a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "addvl x21, x21, #2\n" + "incb x22\n" + "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x28, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162320 // ld1b { za0h.b[x13] }, p0/Z, [x25, x22]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0a982a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x8\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162320 // ld1b { za0h.b[x13] }, p0/Z, [x25, x22]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x27, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x26, x26, #0x10\n" + ".inst 0xe0a982a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "addvl x21, x21, #2\n" + "incb x27\n" + "incb x22\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x24, 11f\n" + "mov x26, %x[in]\n" + "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x20, [x26, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + "cmp x12, x9\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" + "add x13, x13, #0x4\n" + "blt 9b\n" + "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "add x20, x20, #0x4\n" + "blt 10b\n" + "whilelt p8.b, x27, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x21\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp new file mode 100644 index 0000000000..f5ee261964 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<1, 4, VLType::SME, true>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" + "mov x23, %x[width]\n" + "mov z18.b, #0x1\n" + "incb x23\n" + "mov x20, %x[width]\n" + "mov z17.s, #0x0\n" + "sub x10, x21, #0x1\n" + "cntw x9\n" + "sub x23, x23, #0x1\n" + "ands x10, x20, x10\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x10, x10, x21, NE\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x21, x9, #0x1\n" + "sub x20, x23, #0x1\n" + "add x10, x10, #0x3\n" + "whilelt p9.b, XZR, x22\n" + "whilelt p8.b, x21, x22\n" + "mov x28, #0x0\n" + "ptrue p2.b\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x27, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "lsr x10, x10, #0x2\n" + "sub x26, x9, #0x2\n" + "ptrue p11.s\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x25, %x[row_offset]\n" + "mov x24, %x[out]\n" + "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" + "cbnz %x[first], 1f\n" + "addvl x24, x24, #-1\n" + "ld1w { z17.s }, p2/Z, [x24]\n" + "1:" // K loop: Load row sums: End + "mov x23, %x[in]\n" + "ldr x22, [x23, #0x0]\n" + "mov x12, #0x0\n" + "ldr x21, [x23, #0x8]\n" + "add x23, x23, #0x10\n" + "cbz x26, 3f\n" + "2:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902c0 // ld1b { za0h.b[x12] }, p0/Z, [x22, x25]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0xe01902a4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x21, x25]\n" + "add x12, x12, #0x8\n" + "cmp x12, x26, LSL #2\n" + "ldr x21, [x23, #0x8]\n" + "add x23, x23, #0x10\n" + "blt 2b\n" + "3:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902c0 // ld1b { za0h.b[x12] }, p0/Z, [x22, x25]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "mov x23, %x[in]\n" + ".inst 0xe01902a4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x21, x25]\n" + "ldr x22, [x23, #0x0]\n" + "incb x25\n" + "ldr x21, [x23, #0x8]\n" + "add x23, x23, #0x10\n" + "incb x28\n" + "cbz x20, 9f\n" + "mov x20, x20\n" + "4:" // K loop: Main loop + "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 6f\n" + "5:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0xe01922a6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + "sdot z17.s, z16.b, z18.b\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828830 // mova z16.s, p2/M, za0v.s[x12, #1]\n" + ".inst 0xe0a98301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "sdot z17.s, z16.b, z18.b\n" + "add x23, x23, #0x10\n" + "addvl x24, x24, #2\n" + "add x13, x13, #0x8\n" + "blt 5b\n" + "6:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01922a6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "sdot z17.s, z16.b, z18.b\n" + "mov x23, %x[in]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828830 // mova z16.s, p2/M, za0v.s[x12, #1]\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8700 // st1w { za0v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" + "whilelt p9.b, x28, %x[width]\n" + "incb x28\n" + "add x23, x23, #0x10\n" + ".inst 0xe0a98301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "sdot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #2\n" + "incb x25\n" + "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 8f\n" + "7:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x25]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0xe01922a4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + "sdot z17.s, z16.b, z18.b\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828930 // mova z16.s, p2/M, za2v.s[x12, #1]\n" + ".inst 0xe0a98309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "sdot z17.s, z16.b, z18.b\n" + "add x23, x23, #0x10\n" + "addvl x24, x24, #2\n" + "add x13, x13, #0x8\n" + "blt 7b\n" + "8:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x25]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01922a4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + "sdot z17.s, z16.b, z18.b\n" + "mov x23, %x[in]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828930 // mova z16.s, p2/M, za2v.s[x12, #1]\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8708 // st1w { za2v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" + "whilelt p9.b, x28, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x23, x23, #0x10\n" + ".inst 0xe0a98309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "sdot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #2\n" + "incb x28\n" + "incb x25\n" + "bgt 4b\n" + "9:" // K loop: Tails + "cbnz x27, 12f\n" + "mov x23, %x[in]\n" + "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "ldr x20, [x23, #0x0]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + "sdot z17.s, z16.b, z18.b\n" + ".inst 0xe0192282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x25]\n" + "cmp x12, x9\n" + "add x23, x23, #0x8\n" + "addvl x24, x24, #1\n" + "add x13, x13, #0x4\n" + "blt 10b\n" + "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "11:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "sdot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #1\n" + "add x20, x20, #0x4\n" + "blt 11b\n" + "whilelt p8.b, x28, %x[width]\n" + "b 14f\n" + "12:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "13:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "sdot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #1\n" + "blt 13b\n" + "14:" // K loop: End + "st1w { z17.s }, p2, [x24]\n" + "addvl x24, x24, #1\n" + "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp new file mode 100644 index 0000000000..76c1d053cd --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<1, 4, VLType::SME, false>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" + "mov x23, %x[width]\n" + "incb x23\n" + "mov x20, %x[width]\n" + "sub x10, x21, #0x1\n" + "cntw x9\n" + "sub x23, x23, #0x1\n" + "ands x10, x20, x10\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x10, x10, x21, NE\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x21, x9, #0x1\n" + "sub x20, x23, #0x1\n" + "add x10, x10, #0x3\n" + "sub x28, x9, #0x2\n" + "whilelt p9.b, XZR, x22\n" + "whilelt p8.b, x21, x22\n" + "mov x27, #0x0\n" + "mov x26, %x[in]\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "ldr x25, [x26, #0x0]\n" + "and x24, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "lsr x10, x10, #0x2\n" + "ldr x23, [x26, #0x8]\n" + "ptrue p11.s\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" + "add x26, x26, #0x10\n" + "mov x12, #0x0\n" + "cbz x28, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x22]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0xe01602e4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x23, x22]\n" + "add x12, x12, #0x8\n" + "cmp x12, x28, LSL #2\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x22]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "mov x26, %x[in]\n" + ".inst 0xe01602e4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x23, x22]\n" + "ldr x25, [x26, #0x0]\n" + "incb x22\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "incb x27\n" + "cbz x20, 8f\n" + "mov x20, x20\n" + "3:" // K loop: Main loop + "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x28, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0a982a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x8\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x27, %x[width]\n" + "incb x27\n" + "add x26, x26, #0x10\n" + ".inst 0xe0a982a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "addvl x21, x21, #2\n" + "incb x22\n" + "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x28, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162320 // ld1b { za0h.b[x13] }, p0/Z, [x25, x22]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0a982a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x8\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162320 // ld1b { za0h.b[x13] }, p0/Z, [x25, x22]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x23, x22]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x27, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x26, x26, #0x10\n" + ".inst 0xe0a982a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x9, LSL #2]\n" + "addvl x21, x21, #2\n" + "incb x27\n" + "incb x22\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x24, 11f\n" + "mov x26, %x[in]\n" + "whilelt p8.b, x27, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x20, [x26, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + "cmp x12, x9\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" + "add x13, x13, #0x4\n" + "blt 9b\n" + "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "add x20, x20, #0x4\n" + "blt 10b\n" + "whilelt p8.b, x27, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x21\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp new file mode 100644 index 0000000000..daf2d3a100 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<1, 4, VLType::SME, true>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" + "mov x23, %x[width]\n" + "mov z18.b, #0x1\n" + "incb x23\n" + "mov x20, %x[width]\n" + "mov z17.s, #0x0\n" + "sub x10, x21, #0x1\n" + "cntw x9\n" + "sub x23, x23, #0x1\n" + "ands x10, x20, x10\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x10, x10, x21, NE\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x21, x9, #0x1\n" + "sub x20, x23, #0x1\n" + "add x10, x10, #0x3\n" + "whilelt p9.b, XZR, x22\n" + "whilelt p8.b, x21, x22\n" + "mov x28, #0x0\n" + "ptrue p2.b\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x27, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "lsr x10, x10, #0x2\n" + "sub x26, x9, #0x2\n" + "ptrue p11.s\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x25, %x[row_offset]\n" + "mov x24, %x[out]\n" + "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" + "cbnz %x[first], 1f\n" + "addvl x24, x24, #-1\n" + "ld1w { z17.s }, p2/Z, [x24]\n" + "1:" // K loop: Load row sums: End + "mov x23, %x[in]\n" + "ldr x22, [x23, #0x0]\n" + "mov x12, #0x0\n" + "ldr x21, [x23, #0x8]\n" + "add x23, x23, #0x10\n" + "cbz x26, 3f\n" + "2:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902c0 // ld1b { za0h.b[x12] }, p0/Z, [x22, x25]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0xe01902a4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x21, x25]\n" + "add x12, x12, #0x8\n" + "cmp x12, x26, LSL #2\n" + "ldr x21, [x23, #0x8]\n" + "add x23, x23, #0x10\n" + "blt 2b\n" + "3:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902c0 // ld1b { za0h.b[x12] }, p0/Z, [x22, x25]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "mov x23, %x[in]\n" + ".inst 0xe01902a4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x21, x25]\n" + "ldr x22, [x23, #0x0]\n" + "incb x25\n" + "ldr x21, [x23, #0x8]\n" + "add x23, x23, #0x10\n" + "incb x28\n" + "cbz x20, 9f\n" + "mov x20, x20\n" + "4:" // K loop: Main loop + "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 6f\n" + "5:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0xe01922a6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + "udot z17.s, z16.b, z18.b\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828830 // mova z16.s, p2/M, za0v.s[x12, #1]\n" + ".inst 0xe0a98301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "udot z17.s, z16.b, z18.b\n" + "add x23, x23, #0x10\n" + "addvl x24, x24, #2\n" + "add x13, x13, #0x8\n" + "blt 5b\n" + "6:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01922a6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "udot z17.s, z16.b, z18.b\n" + "mov x23, %x[in]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828830 // mova z16.s, p2/M, za0v.s[x12, #1]\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8700 // st1w { za0v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" + "whilelt p9.b, x28, %x[width]\n" + "incb x28\n" + "add x23, x23, #0x10\n" + ".inst 0xe0a98301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "udot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #2\n" + "incb x25\n" + "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 8f\n" + "7:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x25]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0xe01922a4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + "udot z17.s, z16.b, z18.b\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828930 // mova z16.s, p2/M, za2v.s[x12, #1]\n" + ".inst 0xe0a98309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "udot z17.s, z16.b, z18.b\n" + "add x23, x23, #0x10\n" + "addvl x24, x24, #2\n" + "add x13, x13, #0x8\n" + "blt 7b\n" + "8:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x25]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01922a4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x21, x25]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + "udot z17.s, z16.b, z18.b\n" + "mov x23, %x[in]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x23, #0x0]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0828930 // mova z16.s, p2/M, za2v.s[x12, #1]\n" + "ldr x21, [x23, #0x8]\n" + ".inst 0xe0bf8708 // st1w { za2v.s[x12] }, p1/Z, [x24, XZR, LSL #2]\n" + "whilelt p9.b, x28, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x23, x23, #0x10\n" + ".inst 0xe0a98309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x9, LSL #2]\n" + "udot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #2\n" + "incb x28\n" + "incb x25\n" + "bgt 4b\n" + "9:" // K loop: Tails + "cbnz x27, 12f\n" + "mov x23, %x[in]\n" + "whilelt p8.b, x28, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "ldr x20, [x23, #0x0]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + "udot z17.s, z16.b, z18.b\n" + ".inst 0xe0192282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x25]\n" + "cmp x12, x9\n" + "add x23, x23, #0x8\n" + "addvl x24, x24, #1\n" + "add x13, x13, #0x4\n" + "blt 10b\n" + "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "11:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "udot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #1\n" + "add x20, x20, #0x4\n" + "blt 11b\n" + "whilelt p8.b, x28, %x[width]\n" + "b 14f\n" + "12:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "13:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "udot z17.s, z16.b, z18.b\n" + "addvl x24, x24, #1\n" + "blt 13b\n" + "14:" // K loop: End + "st1w { z17.s }, p2, [x24]\n" + "addvl x24, x24, #1\n" + "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp new file mode 100644 index 0000000000..274f69f370 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<1, 1, VLType::SME, false>( + __fp16 * &out, const __fp16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov x21, %x[width]\n" + "inch x21\n" + "cnth x11\n" + "sub x21, x21, #0x1\n" + "udiv x21, x21, x11\n" // n_passes = ceildiv(width, VL<T>) + "mov x20, %x[width]\n" + "sub x10, x11, #0x1\n" + "sub x9, x21, #0x1\n" + "ands x10, x20, x10\n" + "sub x28, x11, #0x2\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "mov x27, #0x0\n" + "mov x26, %x[in]\n" + "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 + "ldr x25, [x26, #0x0]\n" + "and x24, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "csel x10, x10, x11, NE\n" + "ldr x23, [x26, #0x8]\n" + "ptrue p11.h\n" + "whilelt p10.h, XZR, x20\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "add x26, x26, #0x10\n" + "mov x12, #0x0\n" + "cbz x28, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0xe05602e1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x23, x22, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" + "mov x26, %x[in]\n" + ".inst 0xe05602e1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x23, x22, LSL #1]\n" + "ldr x25, [x26, #0x0]\n" + "inch x22\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "inch x27\n" + "cbz x9, 8f\n" + "mov x20, x9\n" + "3:" // K loop: Main loop + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + "cbz x28, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06b82a1 // st1h { za0v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + "whilelt p9.h, x27, %x[width]\n" + "inch x27\n" + "add x26, x26, #0x10\n" + ".inst 0xe06b82a1 // st1h { za0v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "addvl x21, x21, #2\n" + "inch x22\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + "cbz x28, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06b82a9 // st1h { za1v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560320 // ld1h { za0h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05606e1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x23, x22, LSL #1]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + "whilelt p9.h, x27, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x26, x26, #0x10\n" + ".inst 0xe06b82a9 // st1h { za1v.h[x12, #1] }, p0/Z, [x21, x11, LSL #1]\n" + "addvl x21, x21, #2\n" + "inch x27\n" + "inch x22\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x24, 11f\n" + "mov x26, %x[in]\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + "ldr x20, [x26, #0x0]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0560288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x22, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x11\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" + "blt 9b\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f82a8 // st1h { za1v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "blt 10b\n" + "whilelt p8.h, x27, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "addvl x21, x21, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x21\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp new file mode 100644 index 0000000000..ab290649fd --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<1, 1, VLType::SME, false>( + float * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov x22, %x[width]\n" + "incw x22\n" + "cntw x10\n" + "sub x22, x22, #0x1\n" + "udiv x22, x22, x10\n" // n_passes = ceildiv(width, VL<T>) + "mov x21, %x[width]\n" + "sub x9, x10, #0x1\n" + "sub x20, x22, #0x1\n" + "ands x9, x21, x9\n" + "sub x28, x10, #0x2\n" + "mov x27, #0x0\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x24, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x23, [x26, #0x8]\n" + "csel x9, x9, x10, NE\n" + "ptrue p11.s\n" + "whilelt p10.s, XZR, %x[height]\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.s, x27, %x[width]\n" + "whilelt p8.s, x27, %x[width]\n" + "add x26, x26, #0x10\n" + "mov x12, #0x0\n" + "cbz x28, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0x25706140 // psel p0.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0xe09602e1 // ld1w { za0h.s[x12, #1] }, p0/Z, [x23, x22, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0x25706140 // psel p0.s, p8.s/Z, p10.s[w12, #1]\n" + "mov x26, %x[in]\n" + ".inst 0xe09602e1 // ld1w { za0h.s[x12, #1] }, p0/Z, [x23, x22, LSL #2]\n" + "ldr x25, [x26, #0x0]\n" + "incw x22\n" + "ldr x23, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "incw x27\n" + "cbz x20, 8f\n" + "mov x20, x20\n" + "3:" // K loop: Main loop + "whilelt p8.s, x27, %x[width]\n" + "mov x12, #0x0\n" + "cbz x28, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960328 // ld1w { za2h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0x25706141 // psel p1.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe09606e9 // ld1w { za2h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0aa82a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960328 // ld1w { za2h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25706141 // psel p1.s, p8.s/Z, p10.s[w12, #1]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe09606e9 // ld1w { za2h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.s, x27, %x[width]\n" + "incw x27\n" + "add x26, x26, #0x10\n" + ".inst 0xe0aa82a1 // st1w { za0v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "addvl x21, x21, #2\n" + "incw x22\n" + "whilelt p8.s, x27, %x[width]\n" + "mov x12, #0x0\n" + "cbz x28, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0x25706141 // psel p1.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe09606e1 // ld1w { za0h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28\n" + "add x26, x26, #0x10\n" + "addvl x21, x21, #2\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" + "mov x26, %x[in]\n" + "ldr x25, [x26, #0x0]\n" + ".inst 0x25706141 // psel p1.s, p8.s/Z, p10.s[w12, #1]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe09606e1 // ld1w { za0h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" + "ldr x23, [x26, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.s, x27, %x[width]\n" + "subs x20, x20, #0x1\n" + "add x26, x26, #0x10\n" + ".inst 0xe0aa82a9 // st1w { za2v.s[x12, #1] }, p0/Z, [x21, x10, LSL #2]\n" + "addvl x21, x21, #2\n" + "incw x27\n" + "incw x22\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x24, 11f\n" + "mov x26, %x[in]\n" + "whilelt p8.s, x27, %x[width]\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x20, [x26, #0x0]\n" + ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe0960288 // ld1w { za2h.s[x12] }, p0/Z, [x20, x22, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "add x26, x26, #0x8\n" + "addvl x21, x21, #1\n" + "blt 9b\n" + "whilelt p9.s, x27, %x[width]\n" + "whilelt p8.s, x27, %x[width]\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x21, x21, #1\n" + "blt 10b\n" + "whilelt p8.s, x27, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x21, x21, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x21\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp new file mode 100644 index 0000000000..dc6d12b61e --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<2, 1, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x28\n" + "cmp %x[height], x28\n" + "cnth x27\n" + "csel x28, %x[height], x28, LT\n" + "mov x26, #0x0\n" + "ptrue p13.s\n" + "sub x28, x28, #0x1\n" + "whilelt p12.h, XZR, %x[height]\n" + "whilelt p11.h, x27, %x[height]\n" + "mov x25, %x[row_offset]\n" + "mov x24, %x[out]\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "1:" // Width loop + "add x23, %x[in], XZR, LSL #3\n" + "add x20, %x[in], x27, LSL #3\n" + "ldr x22, [x23], #0x8\n" + "mov x12, #0x0\n" + "ldr x21, [x20], #0x8\n" + "cbz x28, 3f\n" + "2:" // Loads: Loop + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05906c0 // ld1h { za0h.h[x12] }, p1/Z, [x22, x25, LSL #1]\n" + "ldr x22, [x23], #0x8\n" + ".inst 0xe05902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28, LSL #1\n" + "ldr x21, [x20], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + "sub x20, %x[width], x26\n" + ".inst 0x25286580 // psel p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05902c0 // ld1h { za0h.h[x12] }, p0/Z, [x22, x25, LSL #1]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + "cmp x20, x27\n" + ".inst 0xe05902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" + "mov x12, #0x0\n" + "csel x20, x20, x27, LT\n" + "4:" // Stores: Loop + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07f8300 // st1h { za0v.h[x12] }, p0/Z, [x24, XZR, LSL #1]\n" + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07b8308 // st1h { za1v.h[x12] }, p0/Z, [x24, x27, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x20\n" + "addvl x24, x24, #4\n" + "blt 4b\n" + "inch x26\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "inch x25\n" + "b.any 1b\n" + "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp new file mode 100644 index 0000000000..d9189258c1 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<2, 2, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x22\n" + "mov x21, %x[width]\n" + "inch x21\n" + "mov x20, %x[width]\n" + "sub x17, x22, #0x1\n" + "sub x21, x21, #0x1\n" + "ands x17, x20, x17\n" + "cntw x16\n" + "udiv x21, x21, x22\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x22, NE\n" + "sub x13, x21, #0x1\n" + "add x17, x17, #0x1\n" + "sub x15, x16, #0x2\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x20, x16, #0x1\n" + "mov x14, #0x0\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + "cntw x28, ALL, MUL #2\n" + "cntw x27, ALL, MUL #3\n" + "ldr x26, [x10, #0x0]\n" + "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x24, [x11, #0x8]\n" + "lsr x17, x17, #0x1\n" + "ptrue p13.s\n" + "ldr x21, [x10, #0x8]\n" + "whilelt p12.h, XZR, x22\n" + "whilelt p11.h, x20, x22\n" + "mov x23, %x[row_offset]\n" + "mov x22, %x[out]\n" + "whilelt p10.h, x14, %x[width]\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + "mov x12, #0x0\n" + "cbz x15, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" + "add x12, x12, #0x4\n" + "cmp x12, x15, LSL #1\n" + "ldr x21, [x10, #0x8]\n" + "add x10, x10, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + "inch x23\n" + "inch x14\n" + "ldr x24, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + "ldr x21, [x10, #0x8]\n" + "add x10, x10, #0x10\n" + "cbz x13, 8f\n" + "mov x20, x13\n" + "3:" // K loop: Main loop + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0x25796162 // psel p2.h, p8.h/Z, p11.h[w13, #3]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572aab // ld1h { za1h.h[x13, #3] }, p2/Z, [x21, x23, LSL #1]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x10, x10, #0x10\n" + "add x13, x13, #0x4\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x22, x22, #4\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0x25796161 // psel p1.h, p8.h/Z, p11.h[w13, #3]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe05726ab // ld1h { za1h.h[x13, #3] }, p1/Z, [x21, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.h, x14, %x[width]\n" + "inch x14\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "addvl x22, x22, #4\n" + "inch x23\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0x25696162 // psel p2.h, p8.h/Z, p11.h[w13, #2]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572aaa // ld1h { za1h.h[x13, #2] }, p2/Z, [x21, x23, LSL #1]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x10, x10, #0x10\n" + "add x13, x13, #0x4\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x22, x22, #4\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe05726aa // ld1h { za1h.h[x13, #2] }, p1/Z, [x21, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08acc // st1w { za3v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.h, x14, %x[width]\n" + "subs x20, x20, #0x1\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "addvl x22, x22, #4\n" + "inch x14\n" + "inch x23\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x25, 11f\n" + "mov x11, %x[in]\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "ldr x21, [x11, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + "cmp x12, x16\n" + ".inst 0xe05726a1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x21, x23, LSL #1]\n" + ".inst 0xe0572289 // ld1h { za1h.h[x13, #1] }, p0/Z, [x20, x23, LSL #1]\n" + "add x11, x11, #0x8\n" + "addvl x22, x22, #2\n" + "add x13, x13, #0x2\n" + "blt 9b\n" + "whilelt p10.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "addvl x22, x22, #2\n" + "add x20, x20, #0x2\n" + "blt 10b\n" + "whilelt p8.h, x14, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "addvl x22, x22, #2\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x22\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp new file mode 100644 index 0000000000..ef787c89b9 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<2, 2, VLType::SME, false>( + __fp16 * &out, const __fp16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x22\n" + "mov x21, %x[width]\n" + "inch x21\n" + "mov x20, %x[width]\n" + "sub x17, x22, #0x1\n" + "sub x21, x21, #0x1\n" + "ands x17, x20, x17\n" + "cntw x16\n" + "udiv x21, x21, x22\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x22, NE\n" + "sub x13, x21, #0x1\n" + "add x17, x17, #0x1\n" + "sub x15, x16, #0x2\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x20, x16, #0x1\n" + "mov x14, #0x0\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + "cntw x28, ALL, MUL #2\n" + "cntw x27, ALL, MUL #3\n" + "ldr x26, [x10, #0x0]\n" + "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x24, [x11, #0x8]\n" + "lsr x17, x17, #0x1\n" + "ptrue p13.s\n" + "ldr x21, [x10, #0x8]\n" + "whilelt p12.h, XZR, x22\n" + "whilelt p11.h, x20, x22\n" + "mov x23, %x[row_offset]\n" + "mov x22, %x[out]\n" + "whilelt p10.h, x14, %x[width]\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + "mov x12, #0x0\n" + "cbz x15, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" + "add x12, x12, #0x4\n" + "cmp x12, x15, LSL #1\n" + "ldr x21, [x10, #0x8]\n" + "add x10, x10, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + "inch x23\n" + "inch x14\n" + "ldr x24, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + "ldr x21, [x10, #0x8]\n" + "add x10, x10, #0x10\n" + "cbz x13, 8f\n" + "mov x20, x13\n" + "3:" // K loop: Main loop + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0x25796162 // psel p2.h, p8.h/Z, p11.h[w13, #3]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572aab // ld1h { za1h.h[x13, #3] }, p2/Z, [x21, x23, LSL #1]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x10, x10, #0x10\n" + "add x13, x13, #0x4\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x22, x22, #4\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0x25796161 // psel p1.h, p8.h/Z, p11.h[w13, #3]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe05726ab // ld1h { za1h.h[x13, #3] }, p1/Z, [x21, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.h, x14, %x[width]\n" + "inch x14\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "addvl x22, x22, #4\n" + "inch x23\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" + ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0x25696162 // psel p2.h, p8.h/Z, p11.h[w13, #2]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0572aaa // ld1h { za1h.h[x13, #2] }, p2/Z, [x21, x23, LSL #1]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x10, x10, #0x10\n" + "add x13, x13, #0x4\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x22, x22, #4\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe05726aa // ld1h { za1h.h[x13, #2] }, p1/Z, [x21, x23, LSL #1]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08acc // st1w { za3v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.h, x14, %x[width]\n" + "subs x20, x20, #0x1\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "addvl x22, x22, #4\n" + "inch x14\n" + "inch x23\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x25, 11f\n" + "mov x11, %x[in]\n" + "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "ldr x21, [x11, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" + "cmp x12, x16\n" + ".inst 0xe05726a1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x21, x23, LSL #1]\n" + ".inst 0xe0572289 // ld1h { za1h.h[x13, #1] }, p0/Z, [x20, x23, LSL #1]\n" + "add x11, x11, #0x8\n" + "addvl x22, x22, #2\n" + "add x13, x13, #0x2\n" + "blt 9b\n" + "whilelt p10.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "addvl x22, x22, #2\n" + "add x20, x20, #0x2\n" + "blt 10b\n" + "whilelt p8.h, x14, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "addvl x22, x22, #2\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x22\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp new file mode 100644 index 0000000000..905c6b41eb --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<2, 4, VLType::SME, false>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" + "mov x23, %x[width]\n" + "incb x23\n" + "mov x20, %x[width]\n" + "sub x17, x21, #0x1\n" + "cntw x16\n" + "sub x23, x23, #0x1\n" + "ands x17, x20, x17\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x21, NE\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x21, x16, #0x1\n" + "sub x20, x23, #0x1\n" + "add x17, x17, #0x3\n" + "sub x15, x16, #0x2\n" + "whilelt p9.b, XZR, x22\n" + "whilelt p8.b, x21, x22\n" + "mov x14, #0x0\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + "cntw x28, ALL, MUL #2\n" + "cntw x27, ALL, MUL #3\n" + "ldr x26, [x10, #0x0]\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x25, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x24, [x11, #0x8]\n" + "lsr x17, x17, #0x2\n" + "ptrue p11.s\n" + "ldr x23, [x10, #0x8]\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + "mov x12, #0x0\n" + "cbz x15, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160120 // ld1b { za0h.b[x12] }, p0/Z, [x9, x22]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0160341 // ld1b { za0h.b[x12, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0160704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x22]\n" + "ldr x24, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + ".inst 0xe01602e5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x23, x22]\n" + "add x12, x12, #0x8\n" + "cmp x12, x15, LSL #2\n" + "ldr x23, [x10, #0x8]\n" + "add x10, x10, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160120 // ld1b { za0h.b[x12] }, p0/Z, [x9, x22]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe0160341 // ld1b { za0h.b[x12, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + ".inst 0xe0160704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x22]\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe01602e5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x23, x22]\n" + "ldr x26, [x10, #0x0]\n" + "incb x22\n" + "incb x14\n" + "ldr x24, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + "ldr x23, [x10, #0x8]\n" + "add x10, x10, #0x10\n" + "cbz x20, 8f\n" + "mov x20, x20\n" + "3:" // K loop: Main loop + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6142 // psel p2.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0162306 // ld1b { za0h.b[x13, #6] }, p0/Z, [x24, x22]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0162ae7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x23, x22]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x10, x10, #0x10\n" + "add x13, x13, #0x8\n" + ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x21, x21, #4\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + "mov x11, %x[in]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162306 // ld1b { za0h.b[x13, #6] }, p0/Z, [x24, x22]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0x257d6141 // psel p1.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e7 // ld1b { za0h.b[x13, #7] }, p1/Z, [x23, x22]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08aa4 // st1w { za1v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + "whilelt p9.b, x14, %x[width]\n" + "incb x14\n" + ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "addvl x21, x21, #4\n" + "incb x22\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162120 // ld1b { za0h.b[x13] }, p0/Z, [x9, x22]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162341 // ld1b { za0h.b[x13, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6142 // psel p2.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0162304 // ld1b { za0h.b[x13, #4] }, p0/Z, [x24, x22]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0162ae5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x23, x22]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x10, x10, #0x10\n" + "add x13, x13, #0x8\n" + ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x21, x21, #4\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162120 // ld1b { za0h.b[x13] }, p0/Z, [x9, x22]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0xe0162341 // ld1b { za0h.b[x13, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + "mov x11, %x[in]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162304 // ld1b { za0h.b[x13, #4] }, p0/Z, [x24, x22]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0x256d6141 // psel p1.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e5 // ld1b { za0h.b[x13, #5] }, p1/Z, [x23, x22]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08aac // st1w { za3v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + "whilelt p9.b, x14, %x[width]\n" + "subs x20, x20, #0x1\n" + ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "addvl x21, x21, #4\n" + "incb x14\n" + "incb x22\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x25, 11f\n" + "mov x11, %x[in]\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + "ldr x20, [x11, #0x0]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" + "add x12, x12, #0x1\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0162283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x22]\n" + "cmp x12, x16\n" + "add x11, x11, #0x8\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x4\n" + "blt 9b\n" + "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "addvl x21, x21, #2\n" + "add x20, x20, #0x4\n" + "blt 10b\n" + "whilelt p8.b, x14, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "addvl x21, x21, #2\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x21\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp new file mode 100644 index 0000000000..c5c5af20e2 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<2, 4, VLType::SME, true>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" + "mov x23, %x[width]\n" + "mov z20.b, #0x1\n" + "incb x23\n" + "mov x20, %x[width]\n" + "mov z19.s, #0x0\n" + "mov z18.s, #0x0\n" + "sub x17, x21, #0x1\n" + "cntw x16\n" + "sub x23, x23, #0x1\n" + "ands x17, x20, x17\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x21, NE\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x21, x16, #0x1\n" + "sub x20, x23, #0x1\n" + "add x17, x17, #0x3\n" + "whilelt p9.b, XZR, x22\n" + "whilelt p8.b, x21, x22\n" + "mov x15, #0x0\n" + "cntw x14, ALL, MUL #2\n" + "cntw x11, ALL, MUL #3\n" + "ptrue p4.b\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x10, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "lsr x17, x17, #0x2\n" + "sub x9, x16, #0x2\n" + "ptrue p11.s\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" + "cbnz %x[first], 1f\n" + "addvl x27, x27, #-2\n" + "ld1w { z19.s }, p4/Z, [x27]\n" + "ld1w { z18.s }, p4/Z, [x27, #1, MUL VL]\n" + "1:" // K loop: Load row sums: End + "mov x26, %x[in]\n" + "add x25, %x[in], x16, LSL #3\n" + "ldr x24, [x26, #0x0]\n" + "ldr x23, [x25, #0x0]\n" + "mov x12, #0x0\n" + "ldr x22, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "ldr x21, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "cbz x9, 3f\n" + "2:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c02e1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" + "ldr x22, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" + "add x12, x12, #0x8\n" + "cmp x12, x9, LSL #2\n" + "ldr x21, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "blt 2b\n" + "3:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe01c02e1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" + "mov x26, %x[in]\n" + "add x25, %x[in], x16, LSL #3\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" + "ldr x23, [x25, #0x0]\n" + "incb x28\n" + "incb x15\n" + "ldr x22, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "ldr x21, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "cbz x20, 9f\n" + "mov x20, x20\n" + "4:" // K loop: Main loop + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 6f\n" + "5:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6142 // psel p2.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c22c6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x22, x28]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x26, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01c2aa7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x21, x28]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe0bf8760 // st1w { za0v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xe0ae8361 // st1w { za0v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829031 // mova z17.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + ".inst 0xc08290b0 // mova z16.s, p4/M, za1v.s[x12, #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + "addvl x27, x27, #4\n" + "add x13, x13, #0x8\n" + "blt 5b\n" + "6:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6140 // psel p0.b, p8.b/Z, p10.b[w13, #7]\n" + ".inst 0xe01c26c6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x22, x28]\n" + "mov x26, %x[in]\n" + "add x25, %x[in], x16, LSL #3\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c22a7 // ld1b { za0h.b[x13, #7] }, p0/Z, [x21, x28]\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "sdot z18.s, z16.b, z20.b\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x26, #0x8]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829031 // mova z17.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe0bf8f60 // st1w { za0v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc08290b0 // mova z16.s, p4/M, za1v.s[x12, #1]\n" + "whilelt p9.b, x15, %x[width]\n" + ".inst 0xe0b08b64 // st1w { za1v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" + "incb x15\n" + "add x26, x26, #0x10\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xe0ae8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" + "add x25, x25, #0x10\n" + "sdot z18.s, z16.b, z20.b\n" + "incb x28\n" + ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + "addvl x27, x27, #4\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 8f\n" + "7:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c2300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x28]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c22e1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6142 // psel p2.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c22c4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x22, x28]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x26, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01c2aa5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x21, x28]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe0bf8768 // st1w { za2v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" + ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xe0ae8369 // st1w { za2v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829131 // mova z17.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + ".inst 0xc08291b0 // mova z16.s, p4/M, za3v.s[x12, #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + "addvl x27, x27, #4\n" + "add x13, x13, #0x8\n" + "blt 7b\n" + "8:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c2300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x28]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0xe01c22e1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" + ".inst 0xe01c26c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x28]\n" + "mov x26, %x[in]\n" + "add x25, %x[in], x16, LSL #3\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" + ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" + ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" + "sdot z18.s, z16.b, z20.b\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x26, #0x8]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829131 // mova z17.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe0bf8f68 // st1w { za2v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc08291b0 // mova z16.s, p4/M, za3v.s[x12, #1]\n" + "whilelt p9.b, x15, %x[width]\n" + ".inst 0xe0b08b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" + "subs x20, x20, #0x1\n" + "add x26, x26, #0x10\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xe0ae8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" + "add x25, x25, #0x10\n" + "sdot z18.s, z16.b, z20.b\n" + "incb x15\n" + ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + "addvl x27, x27, #4\n" + "incb x28\n" + "bgt 4b\n" + "9:" // K loop: Tails + "cbnz x10, 12f\n" + "mov x26, %x[in]\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "ldr x21, [x26, #0x0]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "ldr x20, [x26, x16, LSL #0x3]\n" + ".inst 0xe01c22a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x28]\n" + "add x12, x12, #0x1\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "cmp x12, x16\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xe01c2283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x28]\n" + "add x26, x26, #0x8\n" + "addvl x27, x27, #2\n" + "add x13, x13, #0x4\n" + "blt 10b\n" + "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "11:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + "addvl x27, x27, #2\n" + "add x20, x20, #0x4\n" + "blt 11b\n" + "whilelt p8.b, x15, %x[width]\n" + "b 14f\n" + "12:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "13:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + "addvl x27, x27, #2\n" + "blt 13b\n" + "14:" // K loop: End + "st1w { z19.s }, p4, [x27]\n" + "st1w { z18.s }, p4, [x27, #1, MUL VL]\n" + "addvl x27, x27, #2\n" + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp new file mode 100644 index 0000000000..ce9a0065c7 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<2, 4, VLType::SME, false>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" + "mov x23, %x[width]\n" + "incb x23\n" + "mov x20, %x[width]\n" + "sub x17, x21, #0x1\n" + "cntw x16\n" + "sub x23, x23, #0x1\n" + "ands x17, x20, x17\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x21, NE\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x21, x16, #0x1\n" + "sub x20, x23, #0x1\n" + "add x17, x17, #0x3\n" + "sub x15, x16, #0x2\n" + "whilelt p9.b, XZR, x22\n" + "whilelt p8.b, x21, x22\n" + "mov x14, #0x0\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + "cntw x28, ALL, MUL #2\n" + "cntw x27, ALL, MUL #3\n" + "ldr x26, [x10, #0x0]\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x25, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "ldr x24, [x11, #0x8]\n" + "lsr x17, x17, #0x2\n" + "ptrue p11.s\n" + "ldr x23, [x10, #0x8]\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x22, %x[row_offset]\n" + "mov x21, %x[out]\n" + "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + "mov x12, #0x0\n" + "cbz x15, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160120 // ld1b { za0h.b[x12] }, p0/Z, [x9, x22]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0160341 // ld1b { za0h.b[x12, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0160704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x22]\n" + "ldr x24, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + ".inst 0xe01602e5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x23, x22]\n" + "add x12, x12, #0x8\n" + "cmp x12, x15, LSL #2\n" + "ldr x23, [x10, #0x8]\n" + "add x10, x10, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0160120 // ld1b { za0h.b[x12] }, p0/Z, [x9, x22]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe0160341 // ld1b { za0h.b[x12, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + ".inst 0xe0160704 // ld1b { za0h.b[x12, #4] }, p1/Z, [x24, x22]\n" + "mov x11, %x[in]\n" + "add x10, %x[in], x16, LSL #3\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe01602e5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x23, x22]\n" + "ldr x26, [x10, #0x0]\n" + "incb x22\n" + "incb x14\n" + "ldr x24, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + "ldr x23, [x10, #0x8]\n" + "add x10, x10, #0x10\n" + "cbz x20, 8f\n" + "mov x20, x20\n" + "3:" // K loop: Main loop + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6142 // psel p2.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0162306 // ld1b { za0h.b[x13, #6] }, p0/Z, [x24, x22]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0162ae7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x23, x22]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x10, x10, #0x10\n" + "add x13, x13, #0x8\n" + ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x21, x21, #4\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + "mov x11, %x[in]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162306 // ld1b { za0h.b[x13, #6] }, p0/Z, [x24, x22]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0x257d6141 // psel p1.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e7 // ld1b { za0h.b[x13, #7] }, p1/Z, [x23, x22]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08aa4 // st1w { za1v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + "whilelt p9.b, x14, %x[width]\n" + "incb x14\n" + ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "addvl x21, x21, #4\n" + "incb x22\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x15, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162120 // ld1b { za0h.b[x13] }, p0/Z, [x9, x22]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162341 // ld1b { za0h.b[x13, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6142 // psel p2.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0xe0162304 // ld1b { za0h.b[x13, #4] }, p0/Z, [x24, x22]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0162ae5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x23, x22]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x10, x10, #0x10\n" + "add x13, x13, #0x8\n" + ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x15\n" + "addvl x21, x21, #4\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0162120 // ld1b { za0h.b[x13] }, p0/Z, [x9, x22]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0xe0162341 // ld1b { za0h.b[x13, #1] }, p0/Z, [x26, x22]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + "mov x11, %x[in]\n" + "ldr x9, [x11, #0x0]\n" + ".inst 0xe0162304 // ld1b { za0h.b[x13, #4] }, p0/Z, [x24, x22]\n" + "add x10, %x[in], x16, LSL #3\n" + ".inst 0x256d6141 // psel p1.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x26, [x10, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01626e5 // ld1b { za0h.b[x13, #5] }, p1/Z, [x23, x22]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08aac // st1w { za3v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + "whilelt p9.b, x14, %x[width]\n" + "subs x20, x20, #0x1\n" + ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + "addvl x21, x21, #4\n" + "incb x14\n" + "incb x22\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x25, 11f\n" + "mov x11, %x[in]\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + "ldr x20, [x11, #0x0]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" + "add x12, x12, #0x1\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe0162283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x22]\n" + "cmp x12, x16\n" + "add x11, x11, #0x8\n" + "addvl x21, x21, #2\n" + "add x13, x13, #0x4\n" + "blt 9b\n" + "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "addvl x21, x21, #2\n" + "add x20, x20, #0x4\n" + "blt 10b\n" + "whilelt p8.b, x14, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "addvl x21, x21, #2\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x21\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp new file mode 100644 index 0000000000..7805152656 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<2, 4, VLType::SME, true>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x21\n" + "mov x23, %x[width]\n" + "mov z20.b, #0x1\n" + "incb x23\n" + "mov x20, %x[width]\n" + "mov z19.s, #0x0\n" + "mov z18.s, #0x0\n" + "sub x17, x21, #0x1\n" + "cntw x16\n" + "sub x23, x23, #0x1\n" + "ands x17, x20, x17\n" + "udiv x23, x23, x21\n" // n_passes = ceildiv(width, VL<T>) + "csel x17, x17, x21, NE\n" + "lsl x22, %x[height], #0x1\n" // height * 2 + "lsl x21, x16, #0x1\n" + "sub x20, x23, #0x1\n" + "add x17, x17, #0x3\n" + "whilelt p9.b, XZR, x22\n" + "whilelt p8.b, x21, x22\n" + "mov x15, #0x0\n" + "cntw x14, ALL, MUL #2\n" + "cntw x11, ALL, MUL #3\n" + "ptrue p4.b\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x10, x23, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "lsr x17, x17, #0x2\n" + "sub x9, x16, #0x2\n" + "ptrue p11.s\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" + "cbnz %x[first], 1f\n" + "addvl x27, x27, #-2\n" + "ld1w { z19.s }, p4/Z, [x27]\n" + "ld1w { z18.s }, p4/Z, [x27, #1, MUL VL]\n" + "1:" // K loop: Load row sums: End + "mov x26, %x[in]\n" + "add x25, %x[in], x16, LSL #3\n" + "ldr x24, [x26, #0x0]\n" + "ldr x23, [x25, #0x0]\n" + "mov x12, #0x0\n" + "ldr x22, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "ldr x21, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "cbz x9, 3f\n" + "2:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c02e1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" + "ldr x22, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" + "add x12, x12, #0x8\n" + "cmp x12, x9, LSL #2\n" + "ldr x21, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "blt 2b\n" + "3:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe01c02e1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25646141 // psel p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0x256c6140 // psel p0.b, p8.b/Z, p10.b[w12, #5]\n" + ".inst 0xe01c06c4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x22, x28]\n" + "mov x26, %x[in]\n" + "add x25, %x[in], x16, LSL #3\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c02a5 // ld1b { za0h.b[x12, #5] }, p0/Z, [x21, x28]\n" + "ldr x23, [x25, #0x0]\n" + "incb x28\n" + "incb x15\n" + "ldr x22, [x26, #0x8]\n" + "add x26, x26, #0x10\n" + "ldr x21, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "cbz x20, 9f\n" + "mov x20, x20\n" + "4:" // K loop: Main loop + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 6f\n" + "5:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + ".inst 0x25756140 // psel p0.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6142 // psel p2.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c22c6 // ld1b { za0h.b[x13, #6] }, p0/Z, [x22, x28]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x26, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01c2aa7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x21, x28]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe0bf8760 // st1w { za0v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "udot z18.s, z17.b, z20.b\n" + ".inst 0xe0ae8361 // st1w { za0v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829030 // mova z16.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + ".inst 0xc08290b1 // mova z17.s, p4/M, za1v.s[x12, #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" + "addvl x27, x27, #4\n" + "add x13, x13, #0x8\n" + "blt 5b\n" + "6:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x257d6140 // psel p0.b, p8.b/Z, p10.b[w13, #7]\n" + ".inst 0xe01c26c6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x22, x28]\n" + "mov x26, %x[in]\n" + "add x25, %x[in], x16, LSL #3\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c22a7 // ld1b { za0h.b[x13, #7] }, p0/Z, [x21, x28]\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "udot z18.s, z17.b, z20.b\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x26, #0x8]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829030 // mova z16.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe0bf8f60 // st1w { za0v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc08290b1 // mova z17.s, p4/M, za1v.s[x12, #1]\n" + "whilelt p9.b, x15, %x[width]\n" + ".inst 0xe0b08b64 // st1w { za1v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" + "incb x15\n" + "add x26, x26, #0x10\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xe0ae8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" + "add x25, x25, #0x10\n" + "udot z18.s, z17.b, z20.b\n" + "incb x28\n" + ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + "addvl x27, x27, #4\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 8f\n" + "7:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c2300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x28]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c22e1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25656140 // psel p0.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6142 // psel p2.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c22c4 // ld1b { za0h.b[x13, #4] }, p0/Z, [x22, x28]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x26, #0x8]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01c2aa5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x21, x28]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe0bf8768 // st1w { za2v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" + ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "udot z18.s, z17.b, z20.b\n" + ".inst 0xe0ae8369 // st1w { za2v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + ".inst 0xc08291b1 // mova z17.s, p4/M, za3v.s[x12, #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" + "addvl x27, x27, #4\n" + "add x13, x13, #0x8\n" + "blt 7b\n" + "8:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c2300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x28]\n" + ".inst 0x252d6140 // psel p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0xe01c22e1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x23, x28]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x256d6140 // psel p0.b, p8.b/Z, p10.b[w13, #5]\n" + ".inst 0xe01c26c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x28]\n" + "mov x26, %x[in]\n" + "add x25, %x[in], x16, LSL #3\n" + "ldr x24, [x26, #0x0]\n" + ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" + ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" + ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" + "udot z18.s, z17.b, z20.b\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" + "ldr x22, [x26, #0x8]\n" + ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe0bf8f68 // st1w { za2v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc08291b1 // mova z17.s, p4/M, za3v.s[x12, #1]\n" + "whilelt p9.b, x15, %x[width]\n" + ".inst 0xe0b08b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" + "subs x20, x20, #0x1\n" + "add x26, x26, #0x10\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xe0ae8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" + "add x25, x25, #0x10\n" + "udot z18.s, z17.b, z20.b\n" + "incb x15\n" + ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" + "addvl x27, x27, #4\n" + "incb x28\n" + "bgt 4b\n" + "9:" // K loop: Tails + "cbnz x10, 12f\n" + "mov x26, %x[in]\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "ldr x21, [x26, #0x0]\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "ldr x20, [x26, x16, LSL #0x3]\n" + ".inst 0xe01c22a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x28]\n" + "add x12, x12, #0x1\n" + ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" + "cmp x12, x16\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" + ".inst 0xe01c2283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x28]\n" + "add x26, x26, #0x8\n" + "addvl x27, x27, #2\n" + "add x13, x13, #0x4\n" + "blt 10b\n" + "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" + "mov x20, #0x0\n" + "mov x12, #0x0\n" + "11:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" + "addvl x27, x27, #2\n" + "add x20, x20, #0x4\n" + "blt 11b\n" + "whilelt p8.b, x15, %x[width]\n" + "b 14f\n" + "12:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "13:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x17\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" + "addvl x27, x27, #2\n" + "blt 13b\n" + "14:" // K loop: End + "st1w { z19.s }, p4, [x27]\n" + "st1w { z18.s }, p4, [x27, #1, MUL VL]\n" + "addvl x27, x27, #2\n" + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp new file mode 100644 index 0000000000..96ab55ee06 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<2, 1, VLType::SME, false>( + __fp16 * &out, const __fp16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x28\n" + "cmp %x[height], x28\n" + "cnth x27\n" + "csel x28, %x[height], x28, LT\n" + "mov x26, #0x0\n" + "ptrue p13.s\n" + "sub x28, x28, #0x1\n" + "whilelt p12.h, XZR, %x[height]\n" + "whilelt p11.h, x27, %x[height]\n" + "mov x25, %x[row_offset]\n" + "mov x24, %x[out]\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "1:" // Width loop + "add x23, %x[in], XZR, LSL #3\n" + "add x20, %x[in], x27, LSL #3\n" + "ldr x22, [x23], #0x8\n" + "mov x12, #0x0\n" + "ldr x21, [x20], #0x8\n" + "cbz x28, 3f\n" + "2:" // Loads: Loop + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05906c0 // ld1h { za0h.h[x12] }, p1/Z, [x22, x25, LSL #1]\n" + "ldr x22, [x23], #0x8\n" + ".inst 0xe05902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x28, LSL #1\n" + "ldr x21, [x20], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + "sub x20, %x[width], x26\n" + ".inst 0x25286580 // psel p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05902c0 // ld1h { za0h.h[x12] }, p0/Z, [x22, x25, LSL #1]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + "cmp x20, x27\n" + ".inst 0xe05902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" + "mov x12, #0x0\n" + "csel x20, x20, x27, LT\n" + "4:" // Stores: Loop + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07f8300 // st1h { za0v.h[x12] }, p0/Z, [x24, XZR, LSL #1]\n" + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07b8308 // st1h { za1v.h[x12] }, p0/Z, [x24, x27, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x20\n" + "addvl x24, x24, #4\n" + "blt 4b\n" + "inch x26\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "inch x25\n" + "b.any 1b\n" + "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp new file mode 100644 index 0000000000..ac4b1b5086 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<2, 1, VLType::SME, false>( + float * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov x22, %x[width]\n" + "incw x22\n" + "cntw x16\n" + "sub x22, x22, #0x1\n" + "udiv x22, x22, x16\n" // n_passes = ceildiv(width, VL<T>) + "mov x21, %x[width]\n" + "sub x15, x16, #0x1\n" + "sub x20, x22, #0x1\n" + "ands x15, x21, x15\n" + "sub x14, x16, #0x2\n" + "mov x13, #0x0\n" + "mov x11, %x[in]\n" + "ldr x10, [x11, #0x0]\n" + "add x9, %x[in], x16, LSL #3\n" + "cntw x28, ALL, MUL #2\n" + "ldr x27, [x9, #0x0]\n" + "cntw x26, ALL, MUL #3\n" + "lsr x20, x20, #0x1\n" // n_loops = (n_passes - 1) / 2 + "ldr x25, [x11, #0x8]\n" + "and x24, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "csel x15, x15, x16, NE\n" + "ldr x21, [x9, #0x8]\n" + "ptrue p13.s\n" + "whilelt p12.s, XZR, %x[height]\n" + "whilelt p11.s, x16, %x[height]\n" + "mov x23, %x[row_offset]\n" + "mov x22, %x[out]\n" + "whilelt p10.s, x13, %x[width]\n" + "whilelt p9.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" + "add x11, x11, #0x10\n" + "add x9, x9, #0x10\n" + "mov x12, #0x0\n" + "cbz x14, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + ".inst 0x25706581 // psel p1.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706160 // psel p0.s, p8.s/Z, p11.s[w12, #1]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0xe0970721 // ld1w { za0h.s[x12, #1] }, p1/Z, [x25, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + ".inst 0xe09702a5 // ld1w { za1h.s[x12, #1] }, p0/Z, [x21, x23, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x14\n" + "ldr x21, [x9, #0x8]\n" + "add x9, x9, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + ".inst 0x25706581 // psel p1.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706160 // psel p0.s, p8.s/Z, p11.s[w12, #1]\n" + "mov x11, %x[in]\n" + "add x9, %x[in], x16, LSL #3\n" + ".inst 0xe0970721 // ld1w { za0h.s[x12, #1] }, p1/Z, [x25, x23, LSL #2]\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0xe09702a5 // ld1w { za1h.s[x12, #1] }, p0/Z, [x21, x23, LSL #2]\n" + "ldr x27, [x9, #0x0]\n" + "incw x23\n" + "incw x13\n" + "ldr x25, [x11, #0x8]\n" + "add x11, x11, #0x10\n" + "ldr x21, [x9, #0x8]\n" + "add x9, x9, #0x10\n" + "cbz x20, 8f\n" + "mov x20, x20\n" + "3:" // K loop: Main loop + "whilelt p9.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" + "mov x12, #0x0\n" + "cbz x14, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970548 // ld1w { za2h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0xe097036c // ld1w { za3h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706162 // psel p2.s, p8.s/Z, p11.s[w12, #1]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0970329 // ld1w { za2h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0970aad // ld1w { za3h.s[x12, #1] }, p2/Z, [x21, x23, LSL #2]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x9, x9, #0x10\n" + ".inst 0xe0ba82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x14\n" + "addvl x22, x22, #4\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970548 // ld1w { za2h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + ".inst 0xe097036c // ld1w { za3h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + "mov x11, %x[in]\n" + "add x9, %x[in], x16, LSL #3\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe0970329 // ld1w { za2h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe09706ad // ld1w { za3h.s[x12, #1] }, p1/Z, [x21, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.s, x13, %x[width]\n" + "incw x13\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x9, x9, #0x10\n" + ".inst 0xe0ba82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" + "addvl x22, x22, #4\n" + "incw x23\n" + "whilelt p9.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" + "mov x12, #0x0\n" + "cbz x14, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706162 // psel p2.s, p8.s/Z, p11.s[w12, #1]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0970321 // ld1w { za0h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0970aa5 // ld1w { za1h.s[x12, #1] }, p2/Z, [x21, x23, LSL #2]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x11, x11, #0x10\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x9, x9, #0x10\n" + ".inst 0xe0ba82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x14\n" + "addvl x22, x22, #4\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" + "mov x11, %x[in]\n" + "add x9, %x[in], x16, LSL #3\n" + "ldr x10, [x11, #0x0]\n" + ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe0970321 // ld1w { za0h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" + "ldr x27, [x9, #0x0]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe09706a5 // ld1w { za1h.s[x12, #1] }, p1/Z, [x21, x23, LSL #2]\n" + "ldr x25, [x11, #0x8]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08acc // st1w { za3v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" + "whilelt p10.s, x13, %x[width]\n" + "subs x20, x20, #0x1\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" + "add x11, x11, #0x10\n" + "add x9, x9, #0x10\n" + ".inst 0xe0ba82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" + "addvl x22, x22, #4\n" + "incw x13\n" + "incw x23\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x24, 11f\n" + "mov x11, %x[in]\n" + "whilelt p9.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "ldr x21, [x11, #0x0]\n" + ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" + ".inst 0xe09706a8 // ld1w { za2h.s[x12] }, p1/Z, [x21, x23, LSL #2]\n" + "add x11, x11, #0x8\n" + "addvl x22, x22, #2\n" + ".inst 0xe097028c // ld1w { za3h.s[x12] }, p0/Z, [x20, x23, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x16\n" + "blt 9b\n" + "whilelt p10.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x15\n" + "addvl x22, x22, #2\n" + "blt 10b\n" + "whilelt p8.s, x13, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x15\n" + "addvl x22, x22, #2\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x22\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp new file mode 100644 index 0000000000..2e53475b5c --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<4, 2, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x16\n" + "cntw x15\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" + "whilelt p11.h, XZR, %x[height]\n" + "whilelt p10.h, x15, %x[height]\n" + "whilelt p9.h, x14, %x[height]\n" + "whilelt p8.h, x13, %x[height]\n" + "mov x11, #0x0\n" + "cnth x10\n" + "ptrue p13.s\n" + "sub x16, x16, #0x1\n" + "zip1 p12.h, p11.h, p9.h\n" + "zip1 p11.h, p10.h, p8.h\n" + "mov x9, %x[row_offset]\n" + "mov x28, %x[out]\n" + "whilelt p10.h, x11, %x[width]\n" + "whilelt p9.h, x11, %x[width]\n" + "whilelt p8.h, x11, %x[width]\n" + "1:" // Width loop + "add x27, %x[in], XZR, LSL #3\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x20, %x[in], x13, LSL #3\n" + "ldr x23, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x20], #0x8\n" + "cbz x16, 3f\n" + "2:" // Loads: Loop + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0490720 // ld1h { za0h.h[x12] }, p1/Z, [x25, x9, LSL #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe04902e8 // ld1h { za1h.h[x12] }, p0/Z, [x23, x9, LSL #1]\n" + ".inst 0x25386581 // psel p1.h, p9.h/Z, p12.h[w12, #1]\n" + ".inst 0x25386160 // psel p0.h, p8.h/Z, p11.h[w12, #1]\n" + "ldr x23, [x26], #0x8\n" + ".inst 0xe04906c1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x22, x9, LSL #1]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe04902a9 // ld1h { za1h.h[x12, #1] }, p0/Z, [x21, x9, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x16, LSL #1\n" + "ldr x21, [x20], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe0490720 // ld1h { za0h.h[x12] }, p1/Z, [x25, x9, LSL #1]\n" + "sub x20, %x[width], x11\n" + ".inst 0xe04902e8 // ld1h { za1h.h[x12] }, p0/Z, [x23, x9, LSL #1]\n" + "cmp x20, x10\n" + "csel x20, x20, x10, LT\n" + ".inst 0x25386580 // psel p0.h, p9.h/Z, p12.h[w12, #1]\n" + ".inst 0xe04902c1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x22, x9, LSL #1]\n" + ".inst 0x25386160 // psel p0.h, p8.h/Z, p11.h[w12, #1]\n" + "add x20, x20, #0x1\n" + ".inst 0xe04902a9 // ld1h { za1h.h[x12, #1] }, p0/Z, [x21, x9, LSL #1]\n" + "mov x12, #0x0\n" + "lsr x20, x20, #0x1\n" + "4:" // Stores: Loop + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0af8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x15, LSL #2]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0ae8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x14, LSL #2]\n" + ".inst 0xe0ad838c // st1w { za3v.s[x12] }, p0/Z, [x28, x13, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x20\n" + "addvl x28, x28, #4\n" + "blt 4b\n" + "inch x11\n" + "whilelt p10.h, x11, %x[width]\n" + "whilelt p9.h, x11, %x[width]\n" + "whilelt p8.h, x11, %x[width]\n" + "inch x9\n" + "b.any 1b\n" + "mov %x[out], x28\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp new file mode 100644 index 0000000000..268bdbb924 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<4, 2, VLType::SME, false>( + __fp16 * &out, const __fp16 * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov x17, #0x0\n" + "mov x16, %x[row_offset]\n" + "cntw x15\n" + "cntw x14\n" + "cntw x11, ALL, MUL #2\n" + "cntw x10, ALL, MUL #3\n" + "cmp %x[height], x15\n" + "cnth x9\n" + "csel x15, %x[height], x15, LT\n" + "whilelt p11.h, XZR, %x[height]\n" + "whilelt p10.h, x14, %x[height]\n" + "whilelt p9.h, x11, %x[height]\n" + "whilelt p8.h, x10, %x[height]\n" + "ptrue p13.s\n" + "sub x15, x15, #0x1\n" + "zip1 p12.h, p11.h, p9.h\n" + "zip1 p11.h, p10.h, p8.h\n" + "mov x28, %x[out]\n" + "whilelt p10.h, x17, %x[width]\n" + "whilelt p9.h, x17, %x[width]\n" + "whilelt p8.h, x17, %x[width]\n" + "1:" // Width loop + "add x27, %x[in], XZR, LSL #3\n" + "add x26, %x[in], x14, LSL #3\n" + "add x25, %x[in], x11, LSL #3\n" + "add x20, %x[in], x10, LSL #3\n" + "ldr x24, [x27], #0x8\n" + "mov x13, #0x0\n" + "ldr x23, [x26], #0x8\n" + "ldr x22, [x25], #0x8\n" + "ldr x21, [x20], #0x8\n" + "cbz x15, 3f\n" + "2:" // Loads: Loop + ".inst 0x25296582 // psel p2.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296161 // psel p1.h, p8.h/Z, p11.h[w13]\n" + ".inst 0x25396580 // psel p0.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0xe0502b00 // ld1h { za0h.h[x13] }, p2/Z, [x24, x16, LSL #1]\n" + ".inst 0x25396162 // psel p2.h, p8.h/Z, p11.h[w13, #1]\n" + "ldr x24, [x27], #0x8\n" + ".inst 0xe05026e8 // ld1h { za1h.h[x13] }, p1/Z, [x23, x16, LSL #1]\n" + "ldr x23, [x26], #0x8\n" + ".inst 0xe05022c1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x22, x16, LSL #1]\n" + "ldr x22, [x25], #0x8\n" + ".inst 0xe0502aa9 // ld1h { za1h.h[x13, #1] }, p2/Z, [x21, x16, LSL #1]\n" + "add x13, x13, #0x2\n" + "ldr x21, [x20], #0x8\n" + "cmp x13, x15, LSL #1\n" + "blt 2b\n" + "3:" // Loads: Tail + ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" + "sub x20, %x[width], x17\n" + ".inst 0x25396582 // psel p2.h, p9.h/Z, p12.h[w13, #1]\n" + "cmp x20, x9\n" + "mov x12, #0x0\n" + ".inst 0xe0502700 // ld1h { za0h.h[x13] }, p1/Z, [x24, x16, LSL #1]\n" + ".inst 0xe05022e8 // ld1h { za1h.h[x13] }, p0/Z, [x23, x16, LSL #1]\n" + ".inst 0x25396161 // psel p1.h, p8.h/Z, p11.h[w13, #1]\n" + "csel x20, x20, x9, LT\n" + "add x20, x20, #0x1\n" + ".inst 0xe0502ac1 // ld1h { za0h.h[x13, #1] }, p2/Z, [x22, x16, LSL #1]\n" + "lsr x20, x20, #0x1\n" + ".inst 0xe05026a9 // ld1h { za1h.h[x13, #1] }, p1/Z, [x21, x16, LSL #1]\n" + "4:" // Stores: Loop + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" + ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0ae8b84 // st1w { za1v.s[x12] }, p2/Z, [x28, x14, LSL #2]\n" + ".inst 0xe0ab8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x11, LSL #2]\n" + ".inst 0xe0aa838c // st1w { za3v.s[x12] }, p0/Z, [x28, x10, LSL #2]\n" + "add x12, x12, #0x1\n" + "addvl x28, x28, #4\n" + "cmp x12, x20\n" + "blt 4b\n" + "inch x17\n" + "inch x16\n" + "whilelt p10.h, x17, %x[width]\n" + "whilelt p9.h, x17, %x[width]\n" + "whilelt p8.h, x17, %x[width]\n" + "b.any 1b\n" + "mov %x[out], x28\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp new file mode 100644 index 0000000000..67dd5a9bb7 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<4, 4, VLType::SME, false>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x16\n" + "cntw x15\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" + "whilelt p12.b, XZR, %x[height]\n" + "whilelt p10.b, x15, %x[height]\n" + "whilelt p9.b, x14, %x[height]\n" + "whilelt p8.b, x13, %x[height]\n" + "zip1 p12.b, p12.b, p9.b\n" + "zip1 p10.b, p10.b, p8.b\n" + "mov x11, #0x0\n" + "cntb x10\n" + "ptrue p11.s\n" + "sub x16, x16, #0x1\n" + "zip1 p10.b, p12.b, p10.b\n" + "mov x9, %x[row_offset]\n" + "mov x28, %x[out]\n" + "whilelt p9.b, x11, %x[width]\n" + "whilelt p8.b, x11, %x[width]\n" + "1:" // Width loop + "add x27, %x[in], XZR, LSL #3\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x23, %x[in], x13, LSL #3\n" + "ldr x20, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x23], #0x8\n" + "cbz x16, 3f\n" + "2:" // Loads: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0090320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x9]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe0090281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x9]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0xe00906c2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x22, x9]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe00902a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x9]\n" + "add x12, x12, #0x4\n" + "cmp x12, x16, LSL #2\n" + "ldr x21, [x23], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0090320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x9]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe0090281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x9]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" + "sub x20, %x[width], x11\n" + ".inst 0xe00902c2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x22, x9]\n" + "cmp x20, x10\n" + "csel x20, x20, x10, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "add x20, x20, #0x3\n" + ".inst 0xe00902a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x9]\n" + "mov x12, #0x0\n" + "lsr x20, x20, #0x2\n" + "4:" // Stores: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0af8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x15, LSL #2]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0ae8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x14, LSL #2]\n" + ".inst 0xe0ad838c // st1w { za3v.s[x12] }, p0/Z, [x28, x13, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x20\n" + "addvl x28, x28, #4\n" + "blt 4b\n" + "incb x11\n" + "whilelt p9.b, x11, %x[width]\n" + "whilelt p8.b, x11, %x[width]\n" + "incb x9\n" + "b.any 1b\n" + "mov %x[out], x28\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp new file mode 100644 index 0000000000..21d9378368 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<4, 4, VLType::SME, true>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x16\n" + "cntw x15\n" + "mov z24.b, #0x1\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" + "mov z23.s, #0x0\n" + "mov z22.s, #0x0\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" + "mov z21.s, #0x0\n" + "mov z20.s, #0x0\n" + "whilelt p12.b, XZR, %x[height]\n" + "whilelt p10.b, x15, %x[height]\n" + "whilelt p9.b, x14, %x[height]\n" + "whilelt p8.b, x13, %x[height]\n" + "zip1 p12.b, p12.b, p9.b\n" + "zip1 p10.b, p10.b, p8.b\n" + "ptrue p2.b\n" + "cntb x11\n" + "ptrue p11.s\n" + "sub x16, x16, #0x1\n" + "zip1 p10.b, p12.b, p10.b\n" + "mov x10, %x[row_offset]\n" + "mov x9, %x[out]\n" + "cbnz %x[first], 1f\n" + "addvl x9, x9, #-4\n" + "ld1w { z23.s }, p2/Z, [x9]\n" + "ld1w { z22.s }, p2/Z, [x9, #1, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x9, #3, MUL VL]\n" + "1:" // Initialise row sums: End + "mov x28, #0x0\n" + "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" + "2:" // Width loop + "add x27, %x[in], XZR, LSL #3\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x23, %x[in], x13, LSL #3\n" + "ldr x20, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x23], #0x8\n" + "cbz x16, 4f\n" + "3:" // Loads: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe00a0320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x10]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe00a0281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x10]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0xe00a06c2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x22, x10]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe00a02a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x10]\n" + "add x12, x12, #0x4\n" + "cmp x12, x16, LSL #2\n" + "ldr x21, [x23], #0x8\n" + "blt 3b\n" + "4:" // Loads: Tail + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe00a0320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x10]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe00a0281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x10]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" + "sub x20, %x[width], x28\n" + ".inst 0xe00a02c2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x22, x10]\n" + "cmp x20, x11\n" + "csel x20, x20, x11, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "add x20, x20, #0x3\n" + ".inst 0xe00a02a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x10]\n" + "mov x12, #0x0\n" + "lsr x20, x20, #0x2\n" + "5:" // Stores: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8120 // st1w { za0v.s[x12] }, p0/Z, [x9, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0xe0af8124 // st1w { za1v.s[x12] }, p0/Z, [x9, x15, LSL #2]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828893 // mova z19.s, p2/M, za1v.s[x12]\n" + ".inst 0xe0ae8528 // st1w { za2v.s[x12] }, p1/Z, [x9, x14, LSL #2]\n" + ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" + "sdot z23.s, z17.b, z24.b\n" + ".inst 0xe0ad812c // st1w { za3v.s[x12] }, p0/Z, [x9, x13, LSL #2]\n" + ".inst 0xc0828992 // mova z18.s, p2/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x20\n" + "sdot z22.s, z19.b, z24.b\n" + "sdot z21.s, z16.b, z24.b\n" + "addvl x9, x9, #4\n" + "sdot z20.s, z18.b, z24.b\n" + "blt 5b\n" + "incb x28\n" + "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" + "incb x10\n" + "b.any 2b\n" + "st1w { z23.s }, p2, [x9]\n" + "st1w { z22.s }, p2, [x9, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x9, #2, MUL VL]\n" + "st1w { z20.s }, p2, [x9, #3, MUL VL]\n" + "addvl x9, x9, #4\n" + "mov %x[out], x9\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp new file mode 100644 index 0000000000..f149c93293 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<4, 4, VLType::SME, false>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x16\n" + "cntw x15\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" + "whilelt p12.b, XZR, %x[height]\n" + "whilelt p10.b, x15, %x[height]\n" + "whilelt p9.b, x14, %x[height]\n" + "whilelt p8.b, x13, %x[height]\n" + "zip1 p12.b, p12.b, p9.b\n" + "zip1 p10.b, p10.b, p8.b\n" + "mov x11, #0x0\n" + "cntb x10\n" + "ptrue p11.s\n" + "sub x16, x16, #0x1\n" + "zip1 p10.b, p12.b, p10.b\n" + "mov x9, %x[row_offset]\n" + "mov x28, %x[out]\n" + "whilelt p9.b, x11, %x[width]\n" + "whilelt p8.b, x11, %x[width]\n" + "1:" // Width loop + "add x27, %x[in], XZR, LSL #3\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x23, %x[in], x13, LSL #3\n" + "ldr x20, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x23], #0x8\n" + "cbz x16, 3f\n" + "2:" // Loads: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0090320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x9]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe0090281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x9]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0xe00906c2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x22, x9]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe00902a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x9]\n" + "add x12, x12, #0x4\n" + "cmp x12, x16, LSL #2\n" + "ldr x21, [x23], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0090320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x9]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe0090281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x9]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" + "sub x20, %x[width], x11\n" + ".inst 0xe00902c2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x22, x9]\n" + "cmp x20, x10\n" + "csel x20, x20, x10, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "add x20, x20, #0x3\n" + ".inst 0xe00902a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x9]\n" + "mov x12, #0x0\n" + "lsr x20, x20, #0x2\n" + "4:" // Stores: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0af8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x15, LSL #2]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0ae8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x14, LSL #2]\n" + ".inst 0xe0ad838c // st1w { za3v.s[x12] }, p0/Z, [x28, x13, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x20\n" + "addvl x28, x28, #4\n" + "blt 4b\n" + "incb x11\n" + "whilelt p9.b, x11, %x[width]\n" + "whilelt p8.b, x11, %x[width]\n" + "incb x9\n" + "b.any 1b\n" + "mov %x[out], x28\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp new file mode 100644 index 0000000000..252152e3da --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<4, 4, VLType::SME, true>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x16\n" + "cntw x15\n" + "mov z24.b, #0x1\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" + "mov z23.s, #0x0\n" + "mov z22.s, #0x0\n" + "cmp %x[height], x16\n" + "csel x16, %x[height], x16, LT\n" + "mov z21.s, #0x0\n" + "mov z20.s, #0x0\n" + "whilelt p12.b, XZR, %x[height]\n" + "whilelt p10.b, x15, %x[height]\n" + "whilelt p9.b, x14, %x[height]\n" + "whilelt p8.b, x13, %x[height]\n" + "zip1 p12.b, p12.b, p9.b\n" + "zip1 p10.b, p10.b, p8.b\n" + "ptrue p2.b\n" + "cntb x11\n" + "ptrue p11.s\n" + "sub x16, x16, #0x1\n" + "zip1 p10.b, p12.b, p10.b\n" + "mov x10, %x[row_offset]\n" + "mov x9, %x[out]\n" + "cbnz %x[first], 1f\n" + "addvl x9, x9, #-4\n" + "ld1w { z23.s }, p2/Z, [x9]\n" + "ld1w { z22.s }, p2/Z, [x9, #1, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x9, #3, MUL VL]\n" + "1:" // Initialise row sums: End + "mov x28, #0x0\n" + "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" + "2:" // Width loop + "add x27, %x[in], XZR, LSL #3\n" + "add x26, %x[in], x15, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x14, LSL #3\n" + "add x23, %x[in], x13, LSL #3\n" + "ldr x20, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x23], #0x8\n" + "cbz x16, 4f\n" + "3:" // Loads: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe00a0320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x10]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0xe00a0281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x10]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x20, [x26], #0x8\n" + ".inst 0xe00a06c2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x22, x10]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe00a02a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x10]\n" + "add x12, x12, #0x4\n" + "cmp x12, x16, LSL #2\n" + "ldr x21, [x23], #0x8\n" + "blt 3b\n" + "4:" // Loads: Tail + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe00a0320 // ld1b { za0h.b[x12] }, p0/Z, [x25, x10]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe00a0281 // ld1b { za0h.b[x12, #1] }, p0/Z, [x20, x10]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" + "sub x20, %x[width], x28\n" + ".inst 0xe00a02c2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x22, x10]\n" + "cmp x20, x11\n" + "csel x20, x20, x11, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "add x20, x20, #0x3\n" + ".inst 0xe00a02a3 // ld1b { za0h.b[x12, #3] }, p0/Z, [x21, x10]\n" + "mov x12, #0x0\n" + "lsr x20, x20, #0x2\n" + "5:" // Stores: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8120 // st1w { za0v.s[x12] }, p0/Z, [x9, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" + ".inst 0xe0af8124 // st1w { za1v.s[x12] }, p0/Z, [x9, x15, LSL #2]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828891 // mova z17.s, p2/M, za1v.s[x12]\n" + ".inst 0xe0ae8528 // st1w { za2v.s[x12] }, p1/Z, [x9, x14, LSL #2]\n" + ".inst 0xc0828913 // mova z19.s, p2/M, za2v.s[x12]\n" + "udot z23.s, z16.b, z24.b\n" + ".inst 0xe0ad812c // st1w { za3v.s[x12] }, p0/Z, [x9, x13, LSL #2]\n" + ".inst 0xc0828992 // mova z18.s, p2/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x20\n" + "udot z22.s, z17.b, z24.b\n" + "udot z21.s, z19.b, z24.b\n" + "addvl x9, x9, #4\n" + "udot z20.s, z18.b, z24.b\n" + "blt 5b\n" + "incb x28\n" + "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" + "incb x10\n" + "b.any 2b\n" + "st1w { z23.s }, p2, [x9]\n" + "st1w { z22.s }, p2, [x9, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x9, #2, MUL VL]\n" + "st1w { z20.s }, p2, [x9, #3, MUL VL]\n" + "addvl x9, x9, #4\n" + "mov %x[out], x9\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp new file mode 100644 index 0000000000..b11bb93c42 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +template <> +void interleave_block<4, 1, VLType::SME, false>( + float * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x15\n" + "cmp %x[height], x15\n" + "cntw x14\n" + "cntw x13, ALL, MUL #2\n" + "cntw x11, ALL, MUL #3\n" + "csel x15, %x[height], x15, LT\n" + "mov x10, #0x0\n" + "ptrue p4.s\n" + "sub x15, x15, #0x1\n" + "whilelt p3.s, XZR, %x[height]\n" + "whilelt p15.s, x14, %x[height]\n" + "whilelt p14.s, x13, %x[height]\n" + "whilelt p13.s, x11, %x[height]\n" + "mov x9, %x[row_offset]\n" + "mov x28, %x[out]\n" + "whilelt p12.s, x10, %x[width]\n" + "whilelt p11.s, x10, %x[width]\n" + "whilelt p10.s, x10, %x[width]\n" + "whilelt p9.s, x10, %x[width]\n" + "whilelt p8.s, x10, %x[width]\n" + "1:" // Width loop + "add x27, %x[in], XZR, LSL #3\n" + "add x26, %x[in], x14, LSL #3\n" + "ldr x25, [x27], #0x8\n" + "add x24, %x[in], x13, LSL #3\n" + "add x20, %x[in], x11, LSL #3\n" + "ldr x23, [x26], #0x8\n" + "mov x12, #0x0\n" + "ldr x22, [x24], #0x8\n" + "ldr x21, [x20], #0x8\n" + "cbz x15, 3f\n" + "2:" // Loads: Loop + ".inst 0x25306c60 // psel p0.s, p11.s/Z, p3.s[w12]\n" + ".inst 0x253069e2 // psel p2.s, p10.s/Z, p15.s[w12]\n" + ".inst 0xe0890320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x9, LSL #2]\n" + "ldr x25, [x27], #0x8\n" + ".inst 0x253065c1 // psel p1.s, p9.s/Z, p14.s[w12]\n" + ".inst 0x253061a0 // psel p0.s, p8.s/Z, p13.s[w12]\n" + ".inst 0xe0890ae4 // ld1w { za1h.s[x12] }, p2/Z, [x23, x9, LSL #2]\n" + "ldr x23, [x26], #0x8\n" + ".inst 0xe08906c8 // ld1w { za2h.s[x12] }, p1/Z, [x22, x9, LSL #2]\n" + "ldr x22, [x24], #0x8\n" + ".inst 0xe08902ac // ld1w { za3h.s[x12] }, p0/Z, [x21, x9, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x15\n" + "ldr x21, [x20], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + "sub x20, %x[width], x10\n" + ".inst 0x25306c60 // psel p0.s, p11.s/Z, p3.s[w12]\n" + ".inst 0xe0890320 // ld1w { za0h.s[x12] }, p0/Z, [x25, x9, LSL #2]\n" + ".inst 0x253069e0 // psel p0.s, p10.s/Z, p15.s[w12]\n" + ".inst 0x253065c1 // psel p1.s, p9.s/Z, p14.s[w12]\n" + ".inst 0xe08902e4 // ld1w { za1h.s[x12] }, p0/Z, [x23, x9, LSL #2]\n" + ".inst 0x253061a0 // psel p0.s, p8.s/Z, p13.s[w12]\n" + "cmp x20, x14\n" + ".inst 0xe08906c8 // ld1w { za2h.s[x12] }, p1/Z, [x22, x9, LSL #2]\n" + ".inst 0xe08902ac // ld1w { za3h.s[x12] }, p0/Z, [x21, x9, LSL #2]\n" + "mov x12, #0x0\n" + "csel x20, x20, x14, LT\n" + "4:" // Stores: Loop + ".inst 0x25305180 // psel p0.s, p4.s/Z, p12.s[w12]\n" + ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" + ".inst 0x25305180 // psel p0.s, p4.s/Z, p12.s[w12]\n" + ".inst 0xe0ae8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x14, LSL #2]\n" + ".inst 0x25305181 // psel p1.s, p4.s/Z, p12.s[w12]\n" + ".inst 0x25305180 // psel p0.s, p4.s/Z, p12.s[w12]\n" + ".inst 0xe0ad8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x13, LSL #2]\n" + ".inst 0xe0ab838c // st1w { za3v.s[x12] }, p0/Z, [x28, x11, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x20\n" + "addvl x28, x28, #4\n" + "blt 4b\n" + "incw x10\n" + "whilelt p12.s, x10, %x[width]\n" + "whilelt p11.s, x10, %x[width]\n" + "whilelt p10.s, x10, %x[width]\n" + "whilelt p9.s, x10, %x[width]\n" + "whilelt p8.s, x10, %x[width]\n" + "incw x9\n" + "b.any 1b\n" + "mov %x[out], x28\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(ARM_COMPUTE_ENABLE_SME) |