diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/indirect-interleaves')
27 files changed, 5557 insertions, 0 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list-sve.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list-sve.hpp new file mode 100644 index 0000000000..9d8eb222e6 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list-sve.hpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "sme_interleave1VL_bf16_bf16.hpp" +#include "sme_interleave1VL_block2_bf16_bf16.hpp" +#include "sme_interleave1VL_block4_s8_s8.hpp" +#include "sme_interleave1VL_block4_u8_u8.hpp" +#include "sme_interleave1VL_block4_s8_s8_summing.hpp" +#include "sme_interleave1VL_block4_u8_u8_summing.hpp" +#include "sme_interleave1VL_fp16_fp16.hpp" +#include "sme_interleave1VL_fp32_fp32.hpp" +#include "sme_interleave2VL_block2_bf16_bf16.hpp" +#include "sme_interleave2VL_block2_fp16_fp16.hpp" +#include "sme_interleave2VL_block4_s8_s8.hpp" +#include "sme_interleave2VL_block4_s8_s8_summing.hpp" +#include "sme_interleave2VL_block4_u8_u8.hpp" +#include "sme_interleave2VL_block4_u8_u8_summing.hpp" +#include "sme_interleave2VL_fp16_fp16.hpp" +#include "sme_interleave2VL_bf16_bf16.hpp" +#include "sme_interleave2VL_fp32_fp32.hpp" +#include "sme_interleave4VL_block2_bf16_bf16.hpp" +#include "sme_interleave4VL_block4_s8_s8.hpp" +#include "sme_interleave4VL_block4_u8_u8.hpp" +#include "sme_interleave4VL_block4_s8_s8_summing.hpp" +#include "sme_interleave4VL_block4_u8_u8_summing.hpp" +#include "sme_interleave4VL_fp32_fp32.hpp" + +#include "sme2_interleave1VL_block2_fp32_bf16.hpp" +#include "sme2_interleave2VL_block2_fp32_bf16.hpp" +#include "sme2_interleave4VL_block2_fp32_bf16.hpp" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp new file mode 100644 index 0000000000..c6ff375ea2 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 2, VLType::SME, false>( + bfloat16 * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x21, ALL, MUL #2\n" + "sub x27, %x[width], #0x1\n" + "cntw x20, ALL, MUL #2\n" + "sub x19, x21, #0x1\n" + "whilelt p10.s, XZR, %x[height]\n" + "add x27, x27, x20\n" + "ands x26, %x[width], x19\n" + "udiv x27, x27, x20\n" + "csel x26, x26, x21, NE\n" + "mov x25, #0x0\n" + "and x24, x27, #0x1\n" + "sub x27, x27, #0x1\n" + "add x26, x26, #0x1\n" + "mov x19, %x[width]\n" + "ptrue p0.b\n" + "mov x23, %x[outptr_raw]\n" + "mov x22, %x[row_offset]\n" + "cntw x21\n" + "lsr x27, x27, #0x1\n" + "lsr x26, x26, #0x1\n" + "mov x12, #0x0\n" + ".inst 0x25b34731 // whilelt pn9.s, x25, x19, VLx2\n" + "mov x20, %x[in]\n" + "1:" // Width loop: Preamble: Loop + "ldr x19, [x20], #0x8\n" + ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" + ".inst 0xa0164266 // ld1w { z6.s-z7.s }, pn8.s/Z, [x19, x22, LSL #2]\n" + ".inst 0xc160e0c6 // bfcvt z6.h, { z6.s-z7.s }\n" + ".inst 0xc08000c0 // mova za0h.s[x12], p0/M, z6.s\n" + "add x12, x12, #0x1\n" + "cmp x12, x21\n" + "blt 1b\n" + "incw x22, ALL, MUL #2\n" + "incw x25, ALL, MUL #2\n" + "cbz x27, 5f\n" + "2:" // Width loop + "mov x19, %x[width]\n" + "mov x12, #0x0\n" + ".inst 0x25b34731 // whilelt pn9.s, x25, x19, VLx2\n" + "mov x20, %x[in]\n" + "3:" // Width loop: Odd: Loop + "ldr x19, [x20], #0x8\n" + ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" + ".inst 0xa016427e // ld1w { z30.s-z31.s }, pn8.s/Z, [x19, x22, LSL #2]\n" + ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" + ".inst 0xc08003c8 // mova za2h.s[x12], p0/M, z30.s\n" + ".inst 0xc082800f // mova z15.s, p0/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x21\n" + "st1w { z15.s }, p0, [x23]\n" + "addvl x23, x23, #1\n" + "blt 3b\n" + "incw x25, ALL, MUL #2\n" + "mov x19, %x[width]\n" + "incw x22, ALL, MUL #2\n" + "mov x12, #0x0\n" + ".inst 0x25b34731 // whilelt pn9.s, x25, x19, VLx2\n" + "mov x20, %x[in]\n" + "4:" // Width loop: Even: Loop + "ldr x19, [x20], #0x8\n" + ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" + ".inst 0xa0164278 // ld1w { z24.s-z25.s }, pn8.s/Z, [x19, x22, LSL #2]\n" + ".inst 0xc160e318 // bfcvt z24.h, { z24.s-z25.s }\n" + ".inst 0xc0800300 // mova za0h.s[x12], p0/M, z24.s\n" + ".inst 0xc0828110 // mova z16.s, p0/M, za2v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x21\n" + "st1w { z16.s }, p0, [x23]\n" + "addvl x23, x23, #1\n" + "blt 4b\n" + "subs x27, x27, #0x1\n" + "incw x22, ALL, MUL #2\n" + "incw x25, ALL, MUL #2\n" + "bgt 2b\n" + "5:" // Width loop: Tails + "cbnz x24, 8f\n" + "mov x19, %x[width]\n" + "mov x12, #0x0\n" + ".inst 0x25b34731 // whilelt pn9.s, x25, x19, VLx2\n" + "mov x20, %x[in]\n" + "6:" // Width loop: Tails: Even: Odd: Loop + "ldr x19, [x20], #0x8\n" + ".inst 0x25306548 // psel p8.s, p9.s/Z, p10.s[w12]\n" + ".inst 0xa016426e // ld1w { z14.s-z15.s }, pn8.s/Z, [x19, x22, LSL #2]\n" + ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" + ".inst 0xc08001c8 // mova za2h.s[x12], p0/M, z14.s\n" + ".inst 0xc0828010 // mova z16.s, p0/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x21\n" + "st1w { z16.s }, p0, [x23]\n" + "addvl x23, x23, #1\n" + "blt 6b\n" + "mov x12, #0x0\n" + "7:" // Width loop: Tails: Even: Even: Loop + ".inst 0xc0828110 // mova z16.s, p0/M, za2v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x26\n" + "st1w { z16.s }, p0, [x23]\n" + "addvl x23, x23, #1\n" + "blt 7b\n" + "b 10f\n" + "8:" // Width loop: Tails: Odd + "mov x12, #0x0\n" + "9:" // Width loop: Tails: Odd: Loop + ".inst 0xc0828010 // mova z16.s, p0/M, za0v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x26\n" + "st1w { z16.s }, p0, [x23]\n" + "addvl x23, x23, #1\n" + "blt 9b\n" + "10:" // End + "mov %x[outptr_raw], x23\n" + ".inst 0xd503467f // SMSTOP\n" + : [outptr_raw] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp new file mode 100644 index 0000000000..e712eca3ff --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 2, VLType::SME, false>( + bfloat16 * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x21, ALL, MUL #2\n" + "cntw x28\n" + "sub x27, %x[width], #0x1\n" + "cntw x20, ALL, MUL #2\n" + "sub x19, x21, #0x1\n" + ".inst 0x25207815 // ptrue pn13.b\n" + "whilelt p12.s, XZR, %x[height]\n" + "whilelt p11.s, x28, %x[height]\n" + "add x27, x27, x20\n" + "ands x26, %x[width], x19\n" + "udiv x27, x27, x20\n" + "csel x26, x26, x21, NE\n" + "mov x25, #0x0\n" + "and x24, x27, #0x1\n" + "sub x27, x27, #0x1\n" + "add x26, x26, #0x1\n" + "mov x19, %x[width]\n" + "mov x23, %x[in]\n" + "ptrue p0.b\n" + "mov x22, %x[outptr_raw]\n" + "mov x21, %x[row_offset]\n" + "lsr x27, x27, #0x1\n" + "lsr x26, x26, #0x1\n" + "mov x12, #0x0\n" + ".inst 0x25b34732 // whilelt pn10.s, x25, x19, VLx2\n" + "add x20, x23, x28, LSL #3\n" + "1:" // Width loop: Preamble: Loop + "ldr x19, [x23], #0x8\n" + ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" + ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" + ".inst 0xa0154678 // ld1w { z24.s-z25.s }, pn9.s/Z, [x19, x21, LSL #2]\n" + "ldr x19, [x20], #0x8\n" + ".inst 0xa0154276 // ld1w { z22.s-z23.s }, pn8.s/Z, [x19, x21, LSL #2]\n" + ".inst 0xc160e318 // bfcvt z24.h, { z24.s-z25.s }\n" + ".inst 0xc160e2d6 // bfcvt z22.h, { z22.s-z23.s }\n" + ".inst 0xc0800300 // mova za0h.s[x12], p0/M, z24.s\n" + ".inst 0xc08002c4 // mova za1h.s[x12], p0/M, z22.s\n" + "add x12, x12, #0x1\n" + "cmp x12, x28\n" + "blt 1b\n" + "incw x21, ALL, MUL #2\n" + "incw x25, ALL, MUL #2\n" + "cbz x27, 5f\n" + "2:" // Width loop + "mov x19, %x[width]\n" + "mov x23, %x[in]\n" + "mov x12, #0x0\n" + ".inst 0x25b34732 // whilelt pn10.s, x25, x19, VLx2\n" + "add x20, x23, x28, LSL #3\n" + "3:" // Width loop: Odd: Loop + "ldr x19, [x23], #0x8\n" + ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" + ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" + ".inst 0xa0154676 // ld1w { z22.s-z23.s }, pn9.s/Z, [x19, x21, LSL #2]\n" + "ldr x19, [x20], #0x8\n" + ".inst 0xa015426a // ld1w { z10.s-z11.s }, pn8.s/Z, [x19, x21, LSL #2]\n" + ".inst 0xc160e2d6 // bfcvt z22.h, { z22.s-z23.s }\n" + ".inst 0xc160e14a // bfcvt z10.h, { z10.s-z11.s }\n" + ".inst 0xc08002c8 // mova za2h.s[x12], p0/M, z22.s\n" + ".inst 0xc080014c // mova za3h.s[x12], p0/M, z10.s\n" + ".inst 0xc0828008 // mova z8.s, p0/M, za0v.s[x12]\n" + ".inst 0xc0828089 // mova z9.s, p0/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x28\n" + ".inst 0xa06056c8 // st1w { z8.s-z9.s }, pn13.b, [x22]\n" + "addvl x22, x22, #2\n" + "blt 3b\n" + "incw x25, ALL, MUL #2\n" + "mov x19, %x[width]\n" + "mov x23, %x[in]\n" + "incw x21, ALL, MUL #2\n" + "mov x12, #0x0\n" + ".inst 0x25b34732 // whilelt pn10.s, x25, x19, VLx2\n" + "add x20, x23, x28, LSL #3\n" + "4:" // Width loop: Even: Loop + "ldr x19, [x23], #0x8\n" + ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" + ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" + ".inst 0xa015467a // ld1w { z26.s-z27.s }, pn9.s/Z, [x19, x21, LSL #2]\n" + "ldr x19, [x20], #0x8\n" + ".inst 0xa015427e // ld1w { z30.s-z31.s }, pn8.s/Z, [x19, x21, LSL #2]\n" + ".inst 0xc160e35a // bfcvt z26.h, { z26.s-z27.s }\n" + ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" + ".inst 0xc0800340 // mova za0h.s[x12], p0/M, z26.s\n" + ".inst 0xc08003c4 // mova za1h.s[x12], p0/M, z30.s\n" + ".inst 0xc0828106 // mova z6.s, p0/M, za2v.s[x12]\n" + ".inst 0xc082818e // mova z14.s, p0/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x28\n" + ".inst 0xa16056c6 // st1w { z6.s, z14.s }, pn13.b, [x22]\n" + "addvl x22, x22, #2\n" + "blt 4b\n" + "subs x27, x27, #0x1\n" + "incw x21, ALL, MUL #2\n" + "incw x25, ALL, MUL #2\n" + "bgt 2b\n" + "5:" // Width loop: Tails + "cbnz x24, 8f\n" + "mov x19, %x[width]\n" + "mov x23, %x[in]\n" + "mov x12, #0x0\n" + ".inst 0x25b34732 // whilelt pn10.s, x25, x19, VLx2\n" + "add x20, x23, x28, LSL #3\n" + "6:" // Width loop: Tails: Even: Odd: Loop + "ldr x19, [x23], #0x8\n" + ".inst 0x25306989 // psel p9.s, p10.s/Z, p12.s[w12]\n" + ".inst 0x25306968 // psel p8.s, p10.s/Z, p11.s[w12]\n" + ".inst 0xa015466c // ld1w { z12.s-z13.s }, pn9.s/Z, [x19, x21, LSL #2]\n" + "ldr x19, [x20], #0x8\n" + ".inst 0xa015426e // ld1w { z14.s-z15.s }, pn8.s/Z, [x19, x21, LSL #2]\n" + ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" + ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" + ".inst 0xc0800188 // mova za2h.s[x12], p0/M, z12.s\n" + ".inst 0xc08001cc // mova za3h.s[x12], p0/M, z14.s\n" + ".inst 0xc0828007 // mova z7.s, p0/M, za0v.s[x12]\n" + ".inst 0xc082808f // mova z15.s, p0/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x28\n" + ".inst 0xa16056c7 // st1w { z7.s, z15.s }, pn13.b, [x22]\n" + "addvl x22, x22, #2\n" + "blt 6b\n" + "mov x12, #0x0\n" + "7:" // Width loop: Tails: Even: Even: Loop + ".inst 0xc082810e // mova z14.s, p0/M, za2v.s[x12]\n" + ".inst 0xc082818f // mova z15.s, p0/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x26\n" + ".inst 0xa06056ce // st1w { z14.s-z15.s }, pn13.b, [x22]\n" + "addvl x22, x22, #2\n" + "blt 7b\n" + "b 10f\n" + "8:" // Width loop: Tails: Odd + "mov x12, #0x0\n" + "9:" // Width loop: Tails: Odd: Loop + ".inst 0xc0828014 // mova z20.s, p0/M, za0v.s[x12]\n" + ".inst 0xc0828095 // mova z21.s, p0/M, za1v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x26\n" + ".inst 0xa06056d4 // st1w { z20.s-z21.s }, pn13.b, [x22]\n" + "addvl x22, x22, #2\n" + "blt 9b\n" + "10:" // End + "mov %x[outptr_raw], x22\n" + ".inst 0xd503467f // SMSTOP\n" + : [outptr_raw] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp new file mode 100644 index 0000000000..e08d6d992e --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<4, 2, VLType::SME, false>( + bfloat16 * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x22, ALL, MUL #2\n" + "cntw x9\n" + "cntw x21, ALL, MUL #2\n" + "cntw x19, ALL, MUL #3\n" + "sub x20, x22, #0x1\n" + ".inst 0x25207817 // ptrue pn15.b\n" + "whilelt p1.s, XZR, %x[height]\n" + "whilelt p14.s, x9, %x[height]\n" + "whilelt p13.s, x21, %x[height]\n" + "whilelt p12.s, x19, %x[height]\n" + "sub x28, %x[width], #0x1\n" + "cntw x19, ALL, MUL #2\n" + "ands x27, %x[width], x20\n" + "mov x26, %x[in]\n" + "add x28, x28, x19\n" + "csel x27, x27, x22, NE\n" + "add x25, x26, x9, LSL #3\n" + "mov x24, #0x0\n" + "udiv x28, x28, x19\n" + "add x27, x27, #0x1\n" + "mov x19, %x[width]\n" + "add x23, x25, x9, LSL #3\n" + "ptrue p0.b\n" + "mov x22, %x[outptr_raw]\n" + "mov x21, %x[row_offset]\n" + "sub x28, x28, #0x1\n" + "lsr x27, x27, #0x1\n" + "mov x12, #0x0\n" + ".inst 0x25b34713 // whilelt pn11.s, x24, x19, VLx2\n" + "add x20, x23, x9, LSL #3\n" + "1:" // Width loop: Preamble: Loop + "ldr x19, [x26], #0x8\n" + ".inst 0x25306c28 // psel p8.s, p11.s/Z, p1.s[w12]\n" + ".inst 0x25306dca // psel p10.s, p11.s/Z, p14.s[w12]\n" + ".inst 0xa0154278 // ld1w { z24.s-z25.s }, pn8.s/Z, [x19, x21, LSL #2]\n" + "ldr x19, [x25], #0x8\n" + ".inst 0x25306da9 // psel p9.s, p11.s/Z, p13.s[w12]\n" + ".inst 0x25306d88 // psel p8.s, p11.s/Z, p12.s[w12]\n" + ".inst 0xa0154a62 // ld1w { z2.s-z3.s }, pn10.s/Z, [x19, x21, LSL #2]\n" + "ldr x19, [x23], #0x8\n" + ".inst 0xa015466a // ld1w { z10.s-z11.s }, pn9.s/Z, [x19, x21, LSL #2]\n" + ".inst 0xc160e318 // bfcvt z24.h, { z24.s-z25.s }\n" + ".inst 0xc160e042 // bfcvt z2.h, { z2.s-z3.s }\n" + "ldr x19, [x20], #0x8\n" + ".inst 0xa015426c // ld1w { z12.s-z13.s }, pn8.s/Z, [x19, x21, LSL #2]\n" + ".inst 0xc160e14a // bfcvt z10.h, { z10.s-z11.s }\n" + ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" + ".inst 0xc0800300 // mova za0h.s[x12], p0/M, z24.s\n" + ".inst 0xc0800044 // mova za1h.s[x12], p0/M, z2.s\n" + ".inst 0xc0800148 // mova za2h.s[x12], p0/M, z10.s\n" + ".inst 0xc080018c // mova za3h.s[x12], p0/M, z12.s\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "blt 1b\n" + "incw x21, ALL, MUL #2\n" + "incw x24, ALL, MUL #2\n" + "cbz x28, 5f\n" + "2:" // Width loop + "mov x12, #0x0\n" + "3:" // Width loop: Store: Loop + ".inst 0xc0828011 // mova z17.s, p0/M, za0v.s[x12]\n" + ".inst 0xc0828095 // mova z21.s, p0/M, za1v.s[x12]\n" + ".inst 0xc0828119 // mova z25.s, p0/M, za2v.s[x12]\n" + ".inst 0xc082819d // mova z29.s, p0/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + ".inst 0xa160ded1 // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x22]\n" + "addvl x22, x22, #4\n" + "blt 3b\n" + "mov x26, %x[in]\n" + "add x25, x26, x9, LSL #3\n" + "mov x19, %x[width]\n" + "add x23, x25, x9, LSL #3\n" + "mov x12, #0x0\n" + ".inst 0x25b34713 // whilelt pn11.s, x24, x19, VLx2\n" + "add x20, x23, x9, LSL #3\n" + "4:" // Width loop: Load: Loop + "ldr x19, [x26], #0x8\n" + ".inst 0x25306c28 // psel p8.s, p11.s/Z, p1.s[w12]\n" + ".inst 0x25306dca // psel p10.s, p11.s/Z, p14.s[w12]\n" + ".inst 0xa015426c // ld1w { z12.s-z13.s }, pn8.s/Z, [x19, x21, LSL #2]\n" + "ldr x19, [x25], #0x8\n" + ".inst 0x25306da9 // psel p9.s, p11.s/Z, p13.s[w12]\n" + ".inst 0x25306d88 // psel p8.s, p11.s/Z, p12.s[w12]\n" + ".inst 0xa0154a6e // ld1w { z14.s-z15.s }, pn10.s/Z, [x19, x21, LSL #2]\n" + "ldr x19, [x23], #0x8\n" + ".inst 0xa0154672 // ld1w { z18.s-z19.s }, pn9.s/Z, [x19, x21, LSL #2]\n" + ".inst 0xc160e18c // bfcvt z12.h, { z12.s-z13.s }\n" + ".inst 0xc160e1ce // bfcvt z14.h, { z14.s-z15.s }\n" + "ldr x19, [x20], #0x8\n" + ".inst 0xa015427e // ld1w { z30.s-z31.s }, pn8.s/Z, [x19, x21, LSL #2]\n" + ".inst 0xc160e252 // bfcvt z18.h, { z18.s-z19.s }\n" + ".inst 0xc160e3de // bfcvt z30.h, { z30.s-z31.s }\n" + ".inst 0xc0800180 // mova za0h.s[x12], p0/M, z12.s\n" + ".inst 0xc08001c4 // mova za1h.s[x12], p0/M, z14.s\n" + ".inst 0xc0800248 // mova za2h.s[x12], p0/M, z18.s\n" + ".inst 0xc08003cc // mova za3h.s[x12], p0/M, z30.s\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "blt 4b\n" + "subs x28, x28, #0x1\n" + "incw x21, ALL, MUL #2\n" + "incw x24, ALL, MUL #2\n" + "bgt 2b\n" + "5:" // Width loop: Tails + "mov x12, #0x0\n" + "6:" // Width loop: Tails: Loop + ".inst 0xc0828011 // mova z17.s, p0/M, za0v.s[x12]\n" + ".inst 0xc0828095 // mova z21.s, p0/M, za1v.s[x12]\n" + ".inst 0xc0828119 // mova z25.s, p0/M, za2v.s[x12]\n" + ".inst 0xc082819d // mova z29.s, p0/M, za3v.s[x12]\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + ".inst 0xa160ded1 // st1w { z17.s, z21.s, z25.s, z29.s }, pn15.b, [x22]\n" + "addvl x22, x22, #4\n" + "blt 6b\n" + "7:" // End + "mov %x[outptr_raw], x22\n" + ".inst 0xd503467f // SMSTOP\n" + : [outptr_raw] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp new file mode 100644 index 0000000000..3c8c70776a --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 1, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov x20, %x[width]\n" + "inch x20\n" + "cnth x10\n" + "sub x20, x20, #0x1\n" + "udiv x20, x20, x10\n" // n_passes = ceildiv(width, VL<T>) + "mov x19, %x[width]\n" + "sub x9, x10, #0x1\n" + "sub x28, x20, #0x1\n" + "ands x9, x19, x9\n" + "sub x27, x10, #0x2\n" + "lsl x19, %x[height], #0x1\n" // height * 2 + "mov x26, #0x0\n" + "mov x25, %x[in]\n" + "lsr x28, x28, #0x1\n" // n_loops = (n_passes - 1) / 2 + "ldr x24, [x25, #0x0]\n" + "and x23, x20, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "csel x9, x9, x10, NE\n" + "ldr x22, [x25, #0x8]\n" + "ptrue p11.h\n" + "whilelt p10.h, XZR, x19\n" + "mov x21, %x[row_offset]\n" + "mov x20, %x[out]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "add x25, x25, #0x10\n" + "mov x12, #0x0\n" + "cbz x27, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550300 // ld1h { za0h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0xe05502c1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x22, x21, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "ldr x22, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550300 // ld1h { za0h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" + "mov x25, %x[in]\n" + ".inst 0xe05502c1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x22, x21, LSL #1]\n" + "ldr x24, [x25, #0x0]\n" + "inch x21\n" + "ldr x22, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "inch x26\n" + "cbz x28, 8f\n" + "mov x19, x28\n" + "3:" // K loop: Main loop + "whilelt p8.h, x26, %x[width]\n" + "mov x12, #0x0\n" + "cbz x27, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550308 // ld1h { za1h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05506c9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x22, x21, LSL #1]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe07f8280 // st1h { za0v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06a8281 // st1h { za0v.h[x12, #1] }, p0/Z, [x20, x10, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "add x25, x25, #0x10\n" + "addvl x20, x20, #2\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550308 // ld1h { za1h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + "mov x25, %x[in]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05506c9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x22, x21, LSL #1]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe07f8280 // st1h { za0v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + "whilelt p9.h, x26, %x[width]\n" + "inch x26\n" + "add x25, x25, #0x10\n" + ".inst 0xe06a8281 // st1h { za0v.h[x12, #1] }, p0/Z, [x20, x10, LSL #1]\n" + "addvl x20, x20, #2\n" + "inch x21\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x12, #0x0\n" + "cbz x27, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550300 // ld1h { za0h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05506c1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x22, x21, LSL #1]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe07f8288 // st1h { za1v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06a8289 // st1h { za1v.h[x12, #1] }, p0/Z, [x20, x10, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "add x25, x25, #0x10\n" + "addvl x20, x20, #2\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550300 // ld1h { za0h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + "mov x25, %x[in]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05506c1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x22, x21, LSL #1]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe07f8288 // st1h { za1v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + "whilelt p9.h, x26, %x[width]\n" + "subs x19, x19, #0x1\n" + "add x25, x25, #0x10\n" + ".inst 0xe06a8289 // st1h { za1v.h[x12, #1] }, p0/Z, [x20, x10, LSL #1]\n" + "addvl x20, x20, #2\n" + "inch x26\n" + "inch x21\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x23, 11f\n" + "mov x25, %x[in]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f8280 // st1h { za0v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550308 // ld1h { za1h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "add x25, x25, #0x8\n" + "addvl x20, x20, #1\n" + "blt 9b\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f8288 // st1h { za1v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x20, x20, #1\n" + "blt 10b\n" + "whilelt p9.h, x26, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f8280 // st1h { za0v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x20, x20, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x20\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp new file mode 100644 index 0000000000..81b346c9ba --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 2, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x20\n" + "cntw x10\n" + "mov x19, %x[width]\n" + "inch x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x20\n" // n_passes = ceildiv(width, VL<T>) + "sub x9, x19, #0x1\n" + "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x28, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x27, x20, #0x1\n" + "ands x27, x19, x27\n" + "csel x27, x27, x20, NE\n" + "add x27, x27, #0x1\n" + "lsr x27, x27, #0x1\n" + "sub x26, x10, #0x2\n" + "ptrue p11.s\n" + "lsl x19, %x[height], #0x1\n" // height * 2 + "whilelt p10.h, XZR, x19\n" + "mov x25, %x[row_offset]\n" + "mov x24, %x[out]\n" + "mov x23, #0x0\n" + "whilelt p9.h, x23, %x[width]\n" + "whilelt p8.h, x23, %x[width]\n" + "mov x22, %x[in]\n" + "ldr x21, [x22, #0x0]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "mov x12, #0x0\n" + "cbz x26, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286140 // dup p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe05902a0 // ld1h { za0h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" + ".inst 0x25686140 // dup p0.h, p8.h/Z, p10.h[w12, #2]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0590282 // ld1h { za0h.h[x12, #2] }, p0/Z, [x20, x25, LSL #1]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "add x12, x12, #0x4\n" + "cmp x12, x26, LSL #1\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286140 // dup p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe05902a0 // ld1h { za0h.h[x12] }, p0/Z, [x21, x25, LSL #1]\n" + ".inst 0x25686140 // dup p0.h, p8.h/Z, p10.h[w12, #2]\n" + "mov x22, %x[in]\n" + ".inst 0xe0590282 // ld1h { za0h.h[x12, #2] }, p0/Z, [x20, x25, LSL #1]\n" + "ldr x21, [x22, #0x0]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "inch x25\n" + "inch x23\n" + "cbz x9, 8f\n" + "mov x19, x9\n" + "3:" // K loop: Main loop + "whilelt p8.h, x23, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25396140 // dup p0.h, p8.h/Z, p10.h[w13, #1]\n" + ".inst 0xe05922a1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x21, x25, LSL #1]\n" + ".inst 0x25796140 // dup p0.h, p8.h/Z, p10.h[w13, #3]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0592283 // ld1h { za0h.h[x13, #3] }, p0/Z, [x20, x25, LSL #1]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x4\n" + ".inst 0xe0aa8301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "addvl x24, x24, #2\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + "mov x22, %x[in]\n" + ".inst 0x25396140 // dup p0.h, p8.h/Z, p10.h[w13, #1]\n" + ".inst 0xe05922a1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x21, x25, LSL #1]\n" + ".inst 0x25796140 // dup p0.h, p8.h/Z, p10.h[w13, #3]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0592283 // ld1h { za0h.h[x13, #3] }, p0/Z, [x20, x25, LSL #1]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.h, x23, %x[width]\n" + ".inst 0xe0aa8301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "addvl x24, x24, #2\n" + "inch x23\n" + "inch x25\n" + "whilelt p8.h, x23, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25296140 // dup p0.h, p8.h/Z, p10.h[w13]\n" + ".inst 0xe05922a0 // ld1h { za0h.h[x13] }, p0/Z, [x21, x25, LSL #1]\n" + ".inst 0x25696140 // dup p0.h, p8.h/Z, p10.h[w13, #2]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0592282 // ld1h { za0h.h[x13, #2] }, p0/Z, [x20, x25, LSL #1]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x4\n" + ".inst 0xe0aa8309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "addvl x24, x24, #2\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + "mov x22, %x[in]\n" + ".inst 0x25296140 // dup p0.h, p8.h/Z, p10.h[w13]\n" + ".inst 0xe05922a0 // ld1h { za0h.h[x13] }, p0/Z, [x21, x25, LSL #1]\n" + ".inst 0x25696140 // dup p0.h, p8.h/Z, p10.h[w13, #2]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0592282 // ld1h { za0h.h[x13, #2] }, p0/Z, [x20, x25, LSL #1]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.h, x23, %x[width]\n" + ".inst 0xe0aa8309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "addvl x24, x24, #2\n" + "inch x23\n" + "inch x25\n" + "subs x19, x19, #0x1\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x28, 11f\n" + "mov x22, %x[in]\n" + "whilelt p8.h, x23, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25396140 // dup p0.h, p8.h/Z, p10.h[w13, #1]\n" + "addvl x24, x24, #1\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe05922a1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x21, x25, LSL #1]\n" + "add x22, x22, #0x8\n" + "add x13, x13, #0x2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 9b\n" + "whilelt p9.h, x23, %x[width]\n" + "whilelt p8.h, x23, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "addvl x24, x24, #1\n" + "add x19, x19, #0x2\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + "blt 10b\n" + "whilelt p9.h, x23, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "addvl x24, x24, #1\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p8", "p9", "p10", "p11", "x9", "x10", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp new file mode 100644 index 0000000000..bee3cc5649 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 4, VLType::SME, false>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x20\n" + "mov x22, %x[width]\n" + "incb x22\n" + "mov x19, %x[width]\n" + "sub x9, x20, #0x1\n" + "cntw x28\n" + "sub x22, x22, #0x1\n" + "ands x9, x19, x9\n" + "udiv x22, x22, x20\n" // n_passes = ceildiv(width, VL<T>) + "csel x9, x9, x20, NE\n" + "lsl x21, %x[height], #0x1\n" // height * 2 + "lsl x20, x28, #0x1\n" + "sub x19, x22, #0x1\n" + "add x9, x9, #0x3\n" + "sub x27, x28, #0x2\n" + "whilelt p9.b, XZR, x21\n" + "whilelt p8.b, x20, x21\n" + "mov x26, #0x0\n" + "mov x25, %x[in]\n" + "lsr x19, x19, #0x1\n" // n_loops = (n_passes - 1) / 2 + "ldr x24, [x25, #0x0]\n" + "and x23, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "lsr x9, x9, #0x2\n" + "ldr x22, [x25, #0x8]\n" + "ptrue p11.s\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x21, %x[row_offset]\n" + "mov x20, %x[out]\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "add x25, x25, #0x10\n" + "mov x12, #0x0\n" + "cbz x27, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0150300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x21]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0xe01502c4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x22, x21]\n" + "add x12, x12, #0x8\n" + "cmp x12, x27, LSL #2\n" + "ldr x22, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0150300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x21]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "mov x25, %x[in]\n" + ".inst 0xe01502c4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x22, x21]\n" + "ldr x24, [x25, #0x0]\n" + "incb x21\n" + "ldr x22, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "incb x26\n" + "cbz x19, 8f\n" + "mov x19, x19\n" + "3:" // K loop: Main loop + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x27, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0152302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x21]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01526c6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x22, x21]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe0bf8280 // st1w { za0v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0bc8281 // st1w { za0v.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "add x25, x25, #0x10\n" + "addvl x20, x20, #2\n" + "add x13, x13, #0x8\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0152302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x21]\n" + "mov x25, %x[in]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01526c6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x22, x21]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe0bf8280 // st1w { za0v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + "incb x26\n" + "add x25, x25, #0x10\n" + ".inst 0xe0bc8281 // st1w { za0v.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "addvl x20, x20, #2\n" + "incb x21\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x27, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0152300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x21]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01526c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x21]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe0bf8288 // st1w { za2v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0bc8289 // st1w { za2v.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "add x25, x25, #0x10\n" + "addvl x20, x20, #2\n" + "add x13, x13, #0x8\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0152300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x21]\n" + "mov x25, %x[in]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01526c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x21]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe0bf8288 // st1w { za2v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + "subs x19, x19, #0x1\n" + "add x25, x25, #0x10\n" + ".inst 0xe0bc8289 // st1w { za2v.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "addvl x20, x20, #2\n" + "incb x26\n" + "incb x21\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x23, 11f\n" + "mov x25, %x[in]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8280 // st1w { za0v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + "ldr x24, [x25, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + "cmp x12, x28\n" + ".inst 0xe0152302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x21]\n" + "add x25, x25, #0x8\n" + "addvl x20, x20, #1\n" + "add x13, x13, #0x4\n" + "blt 9b\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8288 // st1w { za2v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x20, x20, #1\n" + "add x19, x19, #0x4\n" + "blt 10b\n" + "whilelt p9.b, x26, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8280 // st1w { za0v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x20, x20, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x20\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp new file mode 100644 index 0000000000..3ba1b98b73 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 4, VLType::SME, true>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov z18.b, #0x1\n" + "mov z17.s, #0x0\n" + "cntb x20\n" + "cntw x10\n" + "ptrue p1.b\n" + "mov x19, %x[width]\n" + "incb x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x20\n" // n_passes = ceildiv(width, VL<T>) + "sub x9, x19, #0x1\n" + "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x28, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x27, x20, #0x1\n" + "ands x27, x19, x27\n" + "csel x27, x27, x20, NE\n" + "add x27, x27, #0x3\n" + "lsr x27, x27, #0x2\n" + "sub x26, x10, #0x2\n" + "ptrue p11.s\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "lsl x19, x10, #0x1\n" + "whilelt p9.b, XZR, x20\n" + "whilelt p8.b, x19, x20\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x25, %x[row_offset]\n" + "mov x24, %x[out]\n" + "mov x23, #0x0\n" + "whilelt p9.b, x23, %x[width]\n" + "whilelt p8.b, x23, %x[width]\n" + "cbnz %x[first], 1f\n" + "addvl x24, x24, #-1\n" + "ld1w { z17.s }, p1/Z, [x24]\n" + "1:" // K loop: Load row sums: End + "mov x22, %x[in]\n" + "ldr x21, [x22, #0x0]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "mov x12, #0x0\n" + "cbz x26, 3f\n" + "2:" // K loop: Charge: Loop + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902a0 // ld1b { za0h.b[x12] }, p0/Z, [x21, x25]\n" + ".inst 0x25646140 // dup p0.b, p8.b/Z, p10.b[w12, #4]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0190284 // ld1b { za0h.b[x12, #4] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "add x12, x12, #0x8\n" + "cmp x12, x26, LSL #2\n" + "blt 2b\n" + "3:" // K loop: Charge: End + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902a0 // ld1b { za0h.b[x12] }, p0/Z, [x21, x25]\n" + ".inst 0x25646140 // dup p0.b, p8.b/Z, p10.b[w12, #4]\n" + "mov x22, %x[in]\n" + ".inst 0xe0190284 // ld1b { za0h.b[x12, #4] }, p0/Z, [x20, x25]\n" + "ldr x21, [x22, #0x0]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "incb x25\n" + "incb x23\n" + "cbz x9, 9f\n" + "mov x19, x9\n" + "4:" // K loop: Main loop + "whilelt p8.b, x23, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 6f\n" + "5:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x25]\n" + ".inst 0x25756140 // dup p0.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0192286 // ld1b { za0h.b[x13, #6] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0xc0828410 // mova z16.s, p1/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0aa8301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "sdot z17.s, z16.b, z18.b\n" + ".inst 0xc0828430 // mova z16.s, p1/M, za0v.s[x12, #1]\n" + "addvl x24, x24, #2\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "sdot z17.s, z16.b, z18.b\n" + "blt 5b\n" + "6:" // K loop: Main loop: First: Tail + "mov x22, %x[in]\n" + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x25]\n" + ".inst 0x25756140 // dup p0.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0192286 // ld1b { za0h.b[x13, #6] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0xc0828410 // mova z16.s, p1/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x23, %x[width]\n" + ".inst 0xe0aa8301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "sdot z17.s, z16.b, z18.b\n" + ".inst 0xc0828430 // mova z16.s, p1/M, za0v.s[x12, #1]\n" + "addvl x24, x24, #2\n" + "incb x23\n" + "incb x25\n" + "sdot z17.s, z16.b, z18.b\n" + "whilelt p8.b, x23, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 8f\n" + "7:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922a0 // ld1b { za0h.b[x13] }, p0/Z, [x21, x25]\n" + ".inst 0x25656140 // dup p0.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0192284 // ld1b { za0h.b[x13, #4] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0xc0828510 // mova z16.s, p1/M, za2v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0aa8309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "sdot z17.s, z16.b, z18.b\n" + ".inst 0xc0828530 // mova z16.s, p1/M, za2v.s[x12, #1]\n" + "addvl x24, x24, #2\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "sdot z17.s, z16.b, z18.b\n" + "blt 7b\n" + "8:" // K loop: Main loop: Second: Tail + "mov x22, %x[in]\n" + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922a0 // ld1b { za0h.b[x13] }, p0/Z, [x21, x25]\n" + ".inst 0x25656140 // dup p0.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0192284 // ld1b { za0h.b[x13, #4] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0xc0828510 // mova z16.s, p1/M, za2v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x23, %x[width]\n" + ".inst 0xe0aa8309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "sdot z17.s, z16.b, z18.b\n" + ".inst 0xc0828530 // mova z16.s, p1/M, za2v.s[x12, #1]\n" + "addvl x24, x24, #2\n" + "incb x23\n" + "incb x25\n" + "sdot z17.s, z16.b, z18.b\n" + "subs x19, x19, #0x1\n" + "bgt 4b\n" + "9:" // K loop: Tails + "cbnz x28, 12f\n" + "mov x22, %x[in]\n" + "whilelt p8.b, x23, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: First + ".inst 0xc0828410 // mova z16.s, p1/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + "addvl x24, x24, #1\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe01922a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x25]\n" + "sdot z17.s, z16.b, z18.b\n" + "add x22, x22, #0x8\n" + "add x13, x13, #0x4\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 10b\n" + "whilelt p9.b, x23, %x[width]\n" + "whilelt p8.b, x23, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "11:" // K loop: Tails: Even: Second + ".inst 0xc0828510 // mova z16.s, p1/M, za2v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "addvl x24, x24, #1\n" + "add x19, x19, #0x4\n" + "add x12, x12, #0x1\n" + "sdot z17.s, z16.b, z18.b\n" + "cmp x12, x27\n" + "blt 11b\n" + "whilelt p9.b, x23, %x[width]\n" + "b 14f\n" + "12:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "13:" // K loop: Tails: Odd: Loop + ".inst 0xc0828410 // mova z16.s, p1/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "addvl x24, x24, #1\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + "sdot z17.s, z16.b, z18.b\n" + "blt 13b\n" + "14:" // K loop: End + "st1w { z17.s }, p1, [x24]\n" + "addvl x24, x24, #1\n" + "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p8", "p9", "p10", "p11", "x9", "x10", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp new file mode 100644 index 0000000000..881dfe103e --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 4, VLType::SME, false>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x20\n" + "mov x22, %x[width]\n" + "incb x22\n" + "mov x19, %x[width]\n" + "sub x9, x20, #0x1\n" + "cntw x28\n" + "sub x22, x22, #0x1\n" + "ands x9, x19, x9\n" + "udiv x22, x22, x20\n" // n_passes = ceildiv(width, VL<T>) + "csel x9, x9, x20, NE\n" + "lsl x21, %x[height], #0x1\n" // height * 2 + "lsl x20, x28, #0x1\n" + "sub x19, x22, #0x1\n" + "add x9, x9, #0x3\n" + "sub x27, x28, #0x2\n" + "whilelt p9.b, XZR, x21\n" + "whilelt p8.b, x20, x21\n" + "mov x26, #0x0\n" + "mov x25, %x[in]\n" + "lsr x19, x19, #0x1\n" // n_loops = (n_passes - 1) / 2 + "ldr x24, [x25, #0x0]\n" + "and x23, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "lsr x9, x9, #0x2\n" + "ldr x22, [x25, #0x8]\n" + "ptrue p11.s\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x21, %x[row_offset]\n" + "mov x20, %x[out]\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "add x25, x25, #0x10\n" + "mov x12, #0x0\n" + "cbz x27, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0150300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x21]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0xe01502c4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x22, x21]\n" + "add x12, x12, #0x8\n" + "cmp x12, x27, LSL #2\n" + "ldr x22, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe0150300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x21]\n" + ".inst 0x25646140 // psel p0.b, p8.b/Z, p10.b[w12, #4]\n" + "mov x25, %x[in]\n" + ".inst 0xe01502c4 // ld1b { za0h.b[x12, #4] }, p0/Z, [x22, x21]\n" + "ldr x24, [x25, #0x0]\n" + "incb x21\n" + "ldr x22, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "incb x26\n" + "cbz x19, 8f\n" + "mov x19, x19\n" + "3:" // K loop: Main loop + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x27, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0152302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x21]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01526c6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x22, x21]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe0bf8280 // st1w { za0v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0bc8281 // st1w { za0v.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "add x25, x25, #0x10\n" + "addvl x20, x20, #2\n" + "add x13, x13, #0x8\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0152302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x21]\n" + "mov x25, %x[in]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25756141 // psel p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01526c6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x22, x21]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe0bf8280 // st1w { za0v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + "incb x26\n" + "add x25, x25, #0x10\n" + ".inst 0xe0bc8281 // st1w { za0v.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "addvl x20, x20, #2\n" + "incb x21\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x27, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0152300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x21]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01526c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x21]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe0bf8288 // st1w { za2v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0bc8289 // st1w { za2v.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "add x25, x25, #0x10\n" + "addvl x20, x20, #2\n" + "add x13, x13, #0x8\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25256140 // psel p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe0152300 // ld1b { za0h.b[x13] }, p0/Z, [x24, x21]\n" + "mov x25, %x[in]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25656141 // psel p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe01526c4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x22, x21]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe0bf8288 // st1w { za2v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + "subs x19, x19, #0x1\n" + "add x25, x25, #0x10\n" + ".inst 0xe0bc8289 // st1w { za2v.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "addvl x20, x20, #2\n" + "incb x26\n" + "incb x21\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x23, 11f\n" + "mov x25, %x[in]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8280 // st1w { za0v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + "ldr x24, [x25, #0x0]\n" + "add x12, x12, #0x1\n" + ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" + "cmp x12, x28\n" + ".inst 0xe0152302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x21]\n" + "add x25, x25, #0x8\n" + "addvl x20, x20, #1\n" + "add x13, x13, #0x4\n" + "blt 9b\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8288 // st1w { za2v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x20, x20, #1\n" + "add x19, x19, #0x4\n" + "blt 10b\n" + "whilelt p9.b, x26, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8280 // st1w { za0v.s[x12] }, p0/Z, [x20, XZR, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x20, x20, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x20\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp new file mode 100644 index 0000000000..231d7ae213 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 4, VLType::SME, true>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov z18.b, #0x1\n" + "mov z17.s, #0x0\n" + "cntb x20\n" + "cntw x10\n" + "ptrue p1.b\n" + "mov x19, %x[width]\n" + "incb x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x20\n" // n_passes = ceildiv(width, VL<T>) + "sub x9, x19, #0x1\n" + "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x28, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x27, x20, #0x1\n" + "ands x27, x19, x27\n" + "csel x27, x27, x20, NE\n" + "add x27, x27, #0x3\n" + "lsr x27, x27, #0x2\n" + "sub x26, x10, #0x2\n" + "ptrue p11.s\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "lsl x19, x10, #0x1\n" + "whilelt p9.b, XZR, x20\n" + "whilelt p8.b, x19, x20\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x25, %x[row_offset]\n" + "mov x24, %x[out]\n" + "mov x23, #0x0\n" + "whilelt p9.b, x23, %x[width]\n" + "whilelt p8.b, x23, %x[width]\n" + "cbnz %x[first], 1f\n" + "addvl x24, x24, #-1\n" + "ld1w { z17.s }, p1/Z, [x24]\n" + "1:" // K loop: Load row sums: End + "mov x22, %x[in]\n" + "ldr x21, [x22, #0x0]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "mov x12, #0x0\n" + "cbz x26, 3f\n" + "2:" // K loop: Charge: Loop + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902a0 // ld1b { za0h.b[x12] }, p0/Z, [x21, x25]\n" + ".inst 0x25646140 // dup p0.b, p8.b/Z, p10.b[w12, #4]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0190284 // ld1b { za0h.b[x12, #4] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "add x12, x12, #0x8\n" + "cmp x12, x26, LSL #2\n" + "blt 2b\n" + "3:" // K loop: Charge: End + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01902a0 // ld1b { za0h.b[x12] }, p0/Z, [x21, x25]\n" + ".inst 0x25646140 // dup p0.b, p8.b/Z, p10.b[w12, #4]\n" + "mov x22, %x[in]\n" + ".inst 0xe0190284 // ld1b { za0h.b[x12, #4] }, p0/Z, [x20, x25]\n" + "ldr x21, [x22, #0x0]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "incb x25\n" + "incb x23\n" + "cbz x9, 9f\n" + "mov x19, x9\n" + "4:" // K loop: Main loop + "whilelt p8.b, x23, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 6f\n" + "5:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x25]\n" + ".inst 0x25756140 // dup p0.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0192286 // ld1b { za0h.b[x13, #6] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0xc0828410 // mova z16.s, p1/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0aa8301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "udot z17.s, z16.b, z18.b\n" + ".inst 0xc0828430 // mova z16.s, p1/M, za0v.s[x12, #1]\n" + "addvl x24, x24, #2\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "udot z17.s, z16.b, z18.b\n" + "blt 5b\n" + "6:" // K loop: Main loop: First: Tail + "mov x22, %x[in]\n" + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01922a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x25]\n" + ".inst 0x25756140 // dup p0.b, p8.b/Z, p10.b[w13, #6]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0192286 // ld1b { za0h.b[x13, #6] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0xc0828410 // mova z16.s, p1/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x23, %x[width]\n" + ".inst 0xe0aa8301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "udot z17.s, z16.b, z18.b\n" + ".inst 0xc0828430 // mova z16.s, p1/M, za0v.s[x12, #1]\n" + "addvl x24, x24, #2\n" + "incb x23\n" + "incb x25\n" + "udot z17.s, z16.b, z18.b\n" + "whilelt p8.b, x23, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x26, 8f\n" + "7:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922a0 // ld1b { za0h.b[x13] }, p0/Z, [x21, x25]\n" + ".inst 0x25656140 // dup p0.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0192284 // ld1b { za0h.b[x13, #4] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0xc0828510 // mova z16.s, p1/M, za2v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0aa8309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "udot z17.s, z16.b, z18.b\n" + ".inst 0xc0828530 // mova z16.s, p1/M, za2v.s[x12, #1]\n" + "addvl x24, x24, #2\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "udot z17.s, z16.b, z18.b\n" + "blt 7b\n" + "8:" // K loop: Main loop: Second: Tail + "mov x22, %x[in]\n" + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01922a0 // ld1b { za0h.b[x13] }, p0/Z, [x21, x25]\n" + ".inst 0x25656140 // dup p0.b, p8.b/Z, p10.b[w13, #4]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0192284 // ld1b { za0h.b[x13, #4] }, p0/Z, [x20, x25]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0xc0828510 // mova z16.s, p1/M, za2v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x23, %x[width]\n" + ".inst 0xe0aa8309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "udot z17.s, z16.b, z18.b\n" + ".inst 0xc0828530 // mova z16.s, p1/M, za2v.s[x12, #1]\n" + "addvl x24, x24, #2\n" + "incb x23\n" + "incb x25\n" + "udot z17.s, z16.b, z18.b\n" + "subs x19, x19, #0x1\n" + "bgt 4b\n" + "9:" // K loop: Tails + "cbnz x28, 12f\n" + "mov x22, %x[in]\n" + "whilelt p8.b, x23, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: First + ".inst 0xc0828410 // mova z16.s, p1/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + "addvl x24, x24, #1\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe01922a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x25]\n" + "udot z17.s, z16.b, z18.b\n" + "add x22, x22, #0x8\n" + "add x13, x13, #0x4\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 10b\n" + "whilelt p9.b, x23, %x[width]\n" + "whilelt p8.b, x23, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "11:" // K loop: Tails: Even: Second + ".inst 0xc0828510 // mova z16.s, p1/M, za2v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "addvl x24, x24, #1\n" + "add x19, x19, #0x4\n" + "add x12, x12, #0x1\n" + "udot z17.s, z16.b, z18.b\n" + "cmp x12, x27\n" + "blt 11b\n" + "whilelt p9.b, x23, %x[width]\n" + "b 14f\n" + "12:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "13:" // K loop: Tails: Odd: Loop + ".inst 0xc0828410 // mova z16.s, p1/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "addvl x24, x24, #1\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + "udot z17.s, z16.b, z18.b\n" + "blt 13b\n" + "14:" // K loop: End + "st1w { z17.s }, p1, [x24]\n" + "addvl x24, x24, #1\n" + "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p8", "p9", "p10", "p11", "x9", "x10", "x12", "x13", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp new file mode 100644 index 0000000000..f80ca640ff --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 1, VLType::SME, false>( + __fp16 * &out, const __fp16 * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov x20, %x[width]\n" + "inch x20\n" + "cnth x10\n" + "sub x20, x20, #0x1\n" + "udiv x20, x20, x10\n" // n_passes = ceildiv(width, VL<T>) + "mov x19, %x[width]\n" + "sub x9, x10, #0x1\n" + "sub x28, x20, #0x1\n" + "ands x9, x19, x9\n" + "sub x27, x10, #0x2\n" + "lsl x19, %x[height], #0x1\n" // height * 2 + "mov x26, #0x0\n" + "mov x25, %x[in]\n" + "lsr x28, x28, #0x1\n" // n_loops = (n_passes - 1) / 2 + "ldr x24, [x25, #0x0]\n" + "and x23, x20, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "csel x9, x9, x10, NE\n" + "ldr x22, [x25, #0x8]\n" + "ptrue p11.h\n" + "whilelt p10.h, XZR, x19\n" + "mov x21, %x[row_offset]\n" + "mov x20, %x[out]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "add x25, x25, #0x10\n" + "mov x12, #0x0\n" + "cbz x27, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550300 // ld1h { za0h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0xe05502c1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x22, x21, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "ldr x22, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550300 // ld1h { za0h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + ".inst 0x25386140 // psel p0.h, p8.h/Z, p10.h[w12, #1]\n" + "mov x25, %x[in]\n" + ".inst 0xe05502c1 // ld1h { za0h.h[x12, #1] }, p0/Z, [x22, x21, LSL #1]\n" + "ldr x24, [x25, #0x0]\n" + "inch x21\n" + "ldr x22, [x25, #0x8]\n" + "add x25, x25, #0x10\n" + "inch x26\n" + "cbz x28, 8f\n" + "mov x19, x28\n" + "3:" // K loop: Main loop + "whilelt p8.h, x26, %x[width]\n" + "mov x12, #0x0\n" + "cbz x27, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550308 // ld1h { za1h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05506c9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x22, x21, LSL #1]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe07f8280 // st1h { za0v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06a8281 // st1h { za0v.h[x12, #1] }, p0/Z, [x20, x10, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "add x25, x25, #0x10\n" + "addvl x20, x20, #2\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550308 // ld1h { za1h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + "mov x25, %x[in]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05506c9 // ld1h { za1h.h[x12, #1] }, p1/Z, [x22, x21, LSL #1]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe07f8280 // st1h { za0v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + "whilelt p9.h, x26, %x[width]\n" + "inch x26\n" + "add x25, x25, #0x10\n" + ".inst 0xe06a8281 // st1h { za0v.h[x12, #1] }, p0/Z, [x20, x10, LSL #1]\n" + "addvl x20, x20, #2\n" + "inch x21\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x12, #0x0\n" + "cbz x27, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550300 // ld1h { za0h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05506c1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x22, x21, LSL #1]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe07f8288 // st1h { za1v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + ".inst 0xe06a8289 // st1h { za1v.h[x12, #1] }, p0/Z, [x20, x10, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27\n" + "add x25, x25, #0x10\n" + "addvl x20, x20, #2\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550300 // ld1h { za0h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + "mov x25, %x[in]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25386141 // psel p1.h, p8.h/Z, p10.h[w12, #1]\n" + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe05506c1 // ld1h { za0h.h[x12, #1] }, p1/Z, [x22, x21, LSL #1]\n" + "ldr x22, [x25, #0x8]\n" + ".inst 0xe07f8288 // st1h { za1v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + ".inst 0x25386d20 // psel p0.h, p11.h/Z, p9.h[w12, #1]\n" + "whilelt p9.h, x26, %x[width]\n" + "subs x19, x19, #0x1\n" + "add x25, x25, #0x10\n" + ".inst 0xe06a8289 // st1h { za1v.h[x12, #1] }, p0/Z, [x20, x10, LSL #1]\n" + "addvl x20, x20, #2\n" + "inch x26\n" + "inch x21\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x23, 11f\n" + "mov x25, %x[in]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f8280 // st1h { za0v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + "ldr x24, [x25, #0x0]\n" + ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" + ".inst 0xe0550308 // ld1h { za1h.h[x12] }, p0/Z, [x24, x21, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "add x25, x25, #0x8\n" + "addvl x20, x20, #1\n" + "blt 9b\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f8288 // st1h { za1v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x20, x20, #1\n" + "blt 10b\n" + "whilelt p9.h, x26, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" + ".inst 0xe07f8280 // st1h { za0v.h[x12] }, p0/Z, [x20, XZR, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x9\n" + "addvl x20, x20, #1\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x20\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp new file mode 100644 index 0000000000..874fc797a4 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<1, 1, VLType::SME, false>( + float * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x10\n" + "mov x19, %x[width]\n" + "incw x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x10\n" // n_passes = ceildiv(width, VL<T>) + "sub x9, x19, #0x1\n" + "lsr x9, x9, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x28, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x27, x10, #0x1\n" + "ands x27, x19, x27\n" + "csel x27, x27, x10, NE\n" + "sub x26, x10, #0x2\n" + "ptrue p11.s\n" + "whilelt p10.s, XZR, %x[height]\n" + "mov x25, %x[row_offset]\n" + "mov x24, %x[out]\n" + "mov x23, #0x0\n" + "whilelt p9.s, x23, %x[width]\n" + "whilelt p8.s, x23, %x[width]\n" + "mov x22, %x[in]\n" + "ldr x21, [x22, #0x0]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "mov x12, #0x0\n" + "cbz x26, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25306140 // dup p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe09902a0 // ld1w { za0h.s[x12] }, p0/Z, [x21, x25, LSL #2]\n" + ".inst 0x25706140 // dup p0.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0990281 // ld1w { za0h.s[x12, #1] }, p0/Z, [x20, x25, LSL #2]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25306140 // dup p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe09902a0 // ld1w { za0h.s[x12] }, p0/Z, [x21, x25, LSL #2]\n" + ".inst 0x25706140 // dup p0.s, p8.s/Z, p10.s[w12, #1]\n" + "mov x22, %x[in]\n" + ".inst 0xe0990281 // ld1w { za0h.s[x12, #1] }, p0/Z, [x20, x25, LSL #2]\n" + "ldr x21, [x22, #0x0]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + "incw x25\n" + "incw x23\n" + "cbz x9, 8f\n" + "mov x19, x9\n" + "3:" // K loop: Main loop + "whilelt p8.s, x23, %x[width]\n" + "mov x12, #0x0\n" + "cbz x26, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25306140 // dup p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe09902a8 // ld1w { za2h.s[x12] }, p0/Z, [x21, x25, LSL #2]\n" + ".inst 0x25706140 // dup p0.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0990289 // ld1w { za2h.s[x12, #1] }, p0/Z, [x20, x25, LSL #2]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0aa8301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "addvl x24, x24, #2\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + "mov x22, %x[in]\n" + ".inst 0x25306140 // dup p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe09902a8 // ld1w { za2h.s[x12] }, p0/Z, [x21, x25, LSL #2]\n" + ".inst 0x25706140 // dup p0.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0990289 // ld1w { za2h.s[x12, #1] }, p0/Z, [x20, x25, LSL #2]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.s, x23, %x[width]\n" + ".inst 0xe0aa8301 // st1w { za0v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "addvl x24, x24, #2\n" + "incw x23\n" + "incw x25\n" + "whilelt p8.s, x23, %x[width]\n" + "mov x12, #0x0\n" + "cbz x26, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25306140 // dup p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe09902a0 // ld1w { za0h.s[x12] }, p0/Z, [x21, x25, LSL #2]\n" + ".inst 0x25706140 // dup p0.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0990281 // ld1w { za0h.s[x12, #1] }, p0/Z, [x20, x25, LSL #2]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0aa8309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "addvl x24, x24, #2\n" + "add x12, x12, #0x2\n" + "cmp x12, x26\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + "mov x22, %x[in]\n" + ".inst 0x25306140 // dup p0.s, p8.s/Z, p10.s[w12]\n" + ".inst 0xe09902a0 // ld1w { za0h.s[x12] }, p0/Z, [x21, x25, LSL #2]\n" + ".inst 0x25706140 // dup p0.s, p8.s/Z, p10.s[w12, #1]\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe0990281 // ld1w { za0h.s[x12, #1] }, p0/Z, [x20, x25, LSL #2]\n" + "ldr x20, [x22, #0x8]\n" + "add x22, x22, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.s, x23, %x[width]\n" + ".inst 0xe0aa8309 // st1w { za2v.s[x12, #1] }, p0/Z, [x24, x10, LSL #2]\n" + "addvl x24, x24, #2\n" + "incw x23\n" + "incw x25\n" + "subs x19, x19, #0x1\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x28, 11f\n" + "mov x22, %x[in]\n" + "whilelt p8.s, x23, %x[width]\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + ".inst 0x25306140 // dup p0.s, p8.s/Z, p10.s[w12]\n" + "addvl x24, x24, #1\n" + "ldr x21, [x22, #0x0]\n" + ".inst 0xe09902a8 // ld1w { za2h.s[x12] }, p0/Z, [x21, x25, LSL #2]\n" + "add x22, x22, #0x8\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 9b\n" + "whilelt p9.s, x23, %x[width]\n" + "whilelt p8.s, x23, %x[width]\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8308 // st1w { za2v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "addvl x24, x24, #1\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + "blt 10b\n" + "whilelt p9.s, x23, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" + "addvl x24, x24, #1\n" + "add x12, x12, #0x1\n" + "cmp x12, x27\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x24\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p8", "p9", "p10", "p11", "x9", "x10", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp new file mode 100644 index 0000000000..61fed43394 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 1, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x27\n" + "cmp %x[height], x27\n" + "cnth x26\n" + "csel x27, %x[height], x27, LT\n" + "mov x25, #0x0\n" + "ptrue p13.s\n" + "sub x27, x27, #0x1\n" + "whilelt p12.h, XZR, %x[height]\n" + "whilelt p11.h, x26, %x[height]\n" + "mov x24, %x[row_offset]\n" + "mov x23, %x[out]\n" + "whilelt p10.h, x25, %x[width]\n" + "whilelt p9.h, x25, %x[width]\n" + "whilelt p8.h, x25, %x[width]\n" + "1:" // Width loop + "add x22, %x[in], XZR, LSL #3\n" + "add x19, %x[in], x26, LSL #3\n" + "ldr x21, [x22], #0x8\n" + "mov x12, #0x0\n" + "ldr x20, [x19], #0x8\n" + "cbz x27, 3f\n" + "2:" // Loads: Loop + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05806a0 // ld1h { za0h.h[x12] }, p1/Z, [x21, x24, LSL #1]\n" + "ldr x21, [x22], #0x8\n" + ".inst 0xe0580288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x24, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27, LSL #1\n" + "ldr x20, [x19], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + "sub x19, %x[width], x25\n" + ".inst 0x25286580 // psel p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05802a0 // ld1h { za0h.h[x12] }, p0/Z, [x21, x24, LSL #1]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + "cmp x19, x26\n" + ".inst 0xe0580288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x24, LSL #1]\n" + "mov x12, #0x0\n" + "csel x19, x19, x26, LT\n" + "4:" // Stores: Loop + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07f82e0 // st1h { za0v.h[x12] }, p0/Z, [x23, XZR, LSL #1]\n" + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07a82e8 // st1h { za1v.h[x12] }, p0/Z, [x23, x26, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x19\n" + "addvl x23, x23, #4\n" + "blt 4b\n" + "inch x25\n" + "whilelt p10.h, x25, %x[width]\n" + "whilelt p9.h, x25, %x[width]\n" + "whilelt p8.h, x25, %x[width]\n" + "inch x24\n" + "b.any 1b\n" + "mov %x[out], x23\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp new file mode 100644 index 0000000000..fc7596e67b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 2, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x20\n" + "cntw x16\n" + "cntw x15, ALL, MUL #2\n" + "cntw x14, ALL, MUL #3\n" + "mov x19, %x[width]\n" + "inch x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x20\n" // n_passes = ceildiv(width, VL<T>) + "sub x13, x19, #0x1\n" + "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x11, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x10, x20, #0x1\n" + "ands x10, x19, x10\n" + "csel x10, x10, x20, NE\n" + "add x10, x10, #0x1\n" + "lsr x10, x10, #0x1\n" + "sub x9, x16, #0x2\n" + "ptrue p13.s\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "lsl x19, x16, #0x1\n" + "whilelt p12.h, XZR, x20\n" + "whilelt p11.h, x19, x20\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "mov x26, #0x0\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "mov x12, #0x0\n" + "cbz x9, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286580 // dup p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05c02e0 // ld1h { za0h.h[x12] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25286160 // dup p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05c02c8 // ld1h { za1h.h[x12] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25686580 // dup p0.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0xe05c02a2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25686160 // dup p0.h, p8.h/Z, p11.h[w12, #2]\n" + ".inst 0xe05c028a // ld1h { za1h.h[x12, #2] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "add x12, x12, #0x4\n" + "cmp x12, x9, LSL #1\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286580 // dup p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05c02e0 // ld1h { za0h.h[x12] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25286160 // dup p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05c02c8 // ld1h { za1h.h[x12] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25686580 // dup p0.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0xe05c02a2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25686160 // dup p0.h, p8.h/Z, p11.h[w12, #2]\n" + ".inst 0xe05c028a // ld1h { za1h.h[x12, #2] }, p0/Z, [x20, x28, LSL #1]\n" + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "inch x28\n" + "inch x26\n" + "cbz x13, 8f\n" + "mov x19, x13\n" + "3:" // K loop: Main loop + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25396580 // dup p0.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0xe05c22e1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25396160 // dup p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe05c22c9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25796580 // dup p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0xe05c22a3 // ld1h { za0h.h[x13, #3] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25796160 // dup p0.h, p8.h/Z, p11.h[w13, #3]\n" + ".inst 0xe05c228b // ld1h { za1h.h[x13, #3] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x13, x13, #0x4\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25396580 // dup p0.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0xe05c22e1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25396160 // dup p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe05c22c9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25796580 // dup p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0xe05c22a3 // ld1h { za0h.h[x13, #3] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25796160 // dup p0.h, p8.h/Z, p11.h[w13, #3]\n" + ".inst 0xe05c228b // ld1h { za1h.h[x13, #3] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "whilelt p10.h, x26, %x[width]\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "inch x26\n" + "inch x28\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25296580 // dup p0.h, p9.h/Z, p12.h[w13]\n" + ".inst 0xe05c22e0 // ld1h { za0h.h[x13] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25296160 // dup p0.h, p8.h/Z, p11.h[w13]\n" + ".inst 0xe05c22c8 // ld1h { za1h.h[x13] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25696580 // dup p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0xe05c22a2 // ld1h { za0h.h[x13, #2] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25696160 // dup p0.h, p8.h/Z, p11.h[w13, #2]\n" + ".inst 0xe05c228a // ld1h { za1h.h[x13, #2] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x13, x13, #0x4\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25296580 // dup p0.h, p9.h/Z, p12.h[w13]\n" + ".inst 0xe05c22e0 // ld1h { za0h.h[x13] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25296160 // dup p0.h, p8.h/Z, p11.h[w13]\n" + ".inst 0xe05c22c8 // ld1h { za1h.h[x13] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25696580 // dup p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0xe05c22a2 // ld1h { za0h.h[x13, #2] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25696160 // dup p0.h, p8.h/Z, p11.h[w13, #2]\n" + ".inst 0xe05c228a // ld1h { za1h.h[x13, #2] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "whilelt p10.h, x26, %x[width]\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "inch x26\n" + "inch x28\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "subs x19, x19, #0x1\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x11, 11f\n" + "mov x25, %x[in]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25396581 // dup p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25396160 // dup p0.h, p8.h/Z, p11.h[w13, #1]\n" + "addvl x27, x27, #2\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe05c26e1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x23, x28, LSL #1]\n" + "ldr x22, [x25, x16, LSL #0x3]\n" + ".inst 0xe05c22c9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x22, x28, LSL #1]\n" + "add x12, x12, #0x1\n" + "add x25, x25, #0x8\n" + "add x13, x13, #0x2\n" + "cmp x12, x16\n" + "blt 9b\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + "add x19, x19, #0x2\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 10b\n" + "whilelt p10.h, x26, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p8", "p9", "p10", "p11", "p12", "p13", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp new file mode 100644 index 0000000000..67570a1302 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 2, VLType::SME, false>( + __fp16 * &out, const __fp16 * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x20\n" + "cntw x16\n" + "cntw x15, ALL, MUL #2\n" + "cntw x14, ALL, MUL #3\n" + "mov x19, %x[width]\n" + "inch x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x20\n" // n_passes = ceildiv(width, VL<T>) + "sub x13, x19, #0x1\n" + "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x11, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x10, x20, #0x1\n" + "ands x10, x19, x10\n" + "csel x10, x10, x20, NE\n" + "add x10, x10, #0x1\n" + "lsr x10, x10, #0x1\n" + "sub x9, x16, #0x2\n" + "ptrue p13.s\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "lsl x19, x16, #0x1\n" + "whilelt p12.h, XZR, x20\n" + "whilelt p11.h, x19, x20\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "mov x26, #0x0\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "mov x12, #0x0\n" + "cbz x9, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25286580 // dup p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05c02e0 // ld1h { za0h.h[x12] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25286160 // dup p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05c02c8 // ld1h { za1h.h[x12] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25686580 // dup p0.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0xe05c02a2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25686160 // dup p0.h, p8.h/Z, p11.h[w12, #2]\n" + ".inst 0xe05c028a // ld1h { za1h.h[x12, #2] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "add x12, x12, #0x4\n" + "cmp x12, x9, LSL #1\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25286580 // dup p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05c02e0 // ld1h { za0h.h[x12] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25286160 // dup p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05c02c8 // ld1h { za1h.h[x12] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25686580 // dup p0.h, p9.h/Z, p12.h[w12, #2]\n" + ".inst 0xe05c02a2 // ld1h { za0h.h[x12, #2] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25686160 // dup p0.h, p8.h/Z, p11.h[w12, #2]\n" + ".inst 0xe05c028a // ld1h { za1h.h[x12, #2] }, p0/Z, [x20, x28, LSL #1]\n" + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "inch x28\n" + "inch x26\n" + "cbz x13, 8f\n" + "mov x19, x13\n" + "3:" // K loop: Main loop + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25396580 // dup p0.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0xe05c22e1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25396160 // dup p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe05c22c9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25796580 // dup p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0xe05c22a3 // ld1h { za0h.h[x13, #3] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25796160 // dup p0.h, p8.h/Z, p11.h[w13, #3]\n" + ".inst 0xe05c228b // ld1h { za1h.h[x13, #3] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x13, x13, #0x4\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25396580 // dup p0.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0xe05c22e1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25396160 // dup p0.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe05c22c9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25796580 // dup p0.h, p9.h/Z, p12.h[w13, #3]\n" + ".inst 0xe05c22a3 // ld1h { za0h.h[x13, #3] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25796160 // dup p0.h, p8.h/Z, p11.h[w13, #3]\n" + ".inst 0xe05c228b // ld1h { za1h.h[x13, #3] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "whilelt p10.h, x26, %x[width]\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "inch x26\n" + "inch x28\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25296580 // dup p0.h, p9.h/Z, p12.h[w13]\n" + ".inst 0xe05c22e0 // ld1h { za0h.h[x13] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25296160 // dup p0.h, p8.h/Z, p11.h[w13]\n" + ".inst 0xe05c22c8 // ld1h { za1h.h[x13] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25696580 // dup p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0xe05c22a2 // ld1h { za0h.h[x13, #2] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25696160 // dup p0.h, p8.h/Z, p11.h[w13, #2]\n" + ".inst 0xe05c228a // ld1h { za1h.h[x13, #2] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "add x13, x13, #0x4\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25296580 // dup p0.h, p9.h/Z, p12.h[w13]\n" + ".inst 0xe05c22e0 // ld1h { za0h.h[x13] }, p0/Z, [x23, x28, LSL #1]\n" + ".inst 0x25296160 // dup p0.h, p8.h/Z, p11.h[w13]\n" + ".inst 0xe05c22c8 // ld1h { za1h.h[x13] }, p0/Z, [x22, x28, LSL #1]\n" + ".inst 0x25696580 // dup p0.h, p9.h/Z, p12.h[w13, #2]\n" + ".inst 0xe05c22a2 // ld1h { za0h.h[x13, #2] }, p0/Z, [x21, x28, LSL #1]\n" + ".inst 0x25696160 // dup p0.h, p8.h/Z, p11.h[w13, #2]\n" + ".inst 0xe05c228a // ld1h { za1h.h[x13, #2] }, p0/Z, [x20, x28, LSL #1]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "whilelt p10.h, x26, %x[width]\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "inch x26\n" + "inch x28\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "subs x19, x19, #0x1\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x11, 11f\n" + "mov x25, %x[in]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25396581 // dup p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25396160 // dup p0.h, p8.h/Z, p11.h[w13, #1]\n" + "addvl x27, x27, #2\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe05c26e1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x23, x28, LSL #1]\n" + "ldr x22, [x25, x16, LSL #0x3]\n" + ".inst 0xe05c22c9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x22, x28, LSL #1]\n" + "add x12, x12, #0x1\n" + "add x25, x25, #0x8\n" + "add x13, x13, #0x2\n" + "cmp x12, x16\n" + "blt 9b\n" + "whilelt p10.h, x26, %x[width]\n" + "whilelt p9.h, x26, %x[width]\n" + "whilelt p8.h, x26, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + "add x19, x19, #0x2\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 10b\n" + "whilelt p10.h, x26, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p8", "p9", "p10", "p11", "p12", "p13", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp new file mode 100644 index 0000000000..22f09339b2 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 4, VLType::SME, false>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x20\n" + "cntw x16\n" + "cntw x15, ALL, MUL #2\n" + "cntw x14, ALL, MUL #3\n" + "mov x19, %x[width]\n" + "incb x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x20\n" // n_passes = ceildiv(width, VL<T>) + "sub x13, x19, #0x1\n" + "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x11, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x10, x20, #0x1\n" + "ands x10, x19, x10\n" + "csel x10, x10, x20, NE\n" + "add x10, x10, #0x3\n" + "lsr x10, x10, #0x2\n" + "sub x9, x16, #0x2\n" + "ptrue p11.s\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "lsl x19, x16, #0x1\n" + "whilelt p9.b, XZR, x20\n" + "whilelt p8.b, x19, x20\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "mov x26, #0x0\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "mov x12, #0x0\n" + "cbz x9, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c02e0 // ld1b { za0h.b[x12] }, p0/Z, [x23, x28]\n" + ".inst 0x252c6140 // dup p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25646141 // dup p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0xe01c02c1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256c6140 // dup p0.b, p8.b/Z, p10.b[w12, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c06a4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c0285 // ld1b { za0h.b[x12, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "add x12, x12, #0x8\n" + "cmp x12, x9, LSL #2\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c02e0 // ld1b { za0h.b[x12] }, p0/Z, [x23, x28]\n" + ".inst 0x252c6140 // dup p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25646141 // dup p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0xe01c02c1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256c6140 // dup p0.b, p8.b/Z, p10.b[w12, #5]\n" + "mov x25, %x[in]\n" + ".inst 0xe01c06a4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x21, x28]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c0285 // ld1b { za0h.b[x12, #5] }, p0/Z, [x20, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "incb x28\n" + "incb x26\n" + "cbz x13, 8f\n" + "mov x19, x13\n" + "3:" // K loop: Main loop + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c22e2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x23, x28]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0x25756141 // dup p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + ".inst 0x257d6140 // dup p0.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2287 // ld1b { za0h.b[x13, #7] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c22e2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x23, x28]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0x25756141 // dup p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + ".inst 0x257d6140 // dup p0.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2287 // ld1b { za0h.b[x13, #7] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "incb x26\n" + "incb x28\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c22e0 // ld1b { za0h.b[x13] }, p0/Z, [x23, x28]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25656141 // dup p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01c22c1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256d6140 // dup p0.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2285 // ld1b { za0h.b[x13, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c22e0 // ld1b { za0h.b[x13] }, p0/Z, [x23, x28]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25656141 // dup p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01c22c1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256d6140 // dup p0.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2285 // ld1b { za0h.b[x13, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "incb x26\n" + "incb x28\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "subs x19, x19, #0x1\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x11, 11f\n" + "mov x25, %x[in]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25356141 // dup p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + "addvl x27, x27, #2\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26e2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x23, x28]\n" + "ldr x22, [x25, x16, LSL #0x3]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + "add x25, x25, #0x8\n" + "add x13, x13, #0x4\n" + "add x12, x12, #0x1\n" + "cmp x12, x16\n" + "blt 9b\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "add x19, x19, #0x4\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 10b\n" + "whilelt p9.b, x26, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p8", "p9", "p10", "p11", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp new file mode 100644 index 0000000000..81cde6c8ee --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 4, VLType::SME, true>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov z20.b, #0x1\n" + "mov z19.s, #0x0\n" + "cntb x20\n" + "mov z18.s, #0x0\n" + "cntw x16\n" + "cntw x15, ALL, MUL #2\n" + "cntw x14, ALL, MUL #3\n" + "ptrue p2.b\n" + "mov x19, %x[width]\n" + "incb x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x20\n" // n_passes = ceildiv(width, VL<T>) + "sub x13, x19, #0x1\n" + "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x11, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x10, x20, #0x1\n" + "ands x10, x19, x10\n" + "csel x10, x10, x20, NE\n" + "add x10, x10, #0x3\n" + "lsr x10, x10, #0x2\n" + "sub x9, x16, #0x2\n" + "ptrue p11.s\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "lsl x19, x16, #0x1\n" + "whilelt p9.b, XZR, x20\n" + "whilelt p8.b, x19, x20\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "mov x26, #0x0\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "cbnz %x[first], 1f\n" + "addvl x27, x27, #-2\n" + "ld1w { z19.s }, p2/Z, [x27]\n" + "ld1w { z18.s }, p2/Z, [x27, #1, MUL VL]\n" + "1:" // K loop: Load row sums: End + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "mov x12, #0x0\n" + "cbz x9, 3f\n" + "2:" // K loop: Charge: Loop + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c02e0 // ld1b { za0h.b[x12] }, p0/Z, [x23, x28]\n" + ".inst 0x252c6140 // dup p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25646141 // dup p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0xe01c02c1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256c6140 // dup p0.b, p8.b/Z, p10.b[w12, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c06a4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c0285 // ld1b { za0h.b[x12, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "add x12, x12, #0x8\n" + "cmp x12, x9, LSL #2\n" + "blt 2b\n" + "3:" // K loop: Charge: End + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c02e0 // ld1b { za0h.b[x12] }, p0/Z, [x23, x28]\n" + ".inst 0x252c6140 // dup p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25646141 // dup p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0xe01c02c1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256c6140 // dup p0.b, p8.b/Z, p10.b[w12, #5]\n" + "mov x25, %x[in]\n" + ".inst 0xe01c06a4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x21, x28]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c0285 // ld1b { za0h.b[x12, #5] }, p0/Z, [x20, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "incb x28\n" + "incb x26\n" + "cbz x13, 9f\n" + "mov x19, x13\n" + "4:" // K loop: Main loop + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 6f\n" + "5:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c22e2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x23, x28]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0x25756141 // dup p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + ".inst 0x257d6140 // dup p0.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2287 // ld1b { za0h.b[x13, #7] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0xc0828890 // mova z16.s, p2/M, za1v.s[x12]\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0828831 // mova z17.s, p2/M, za0v.s[x12, #1]\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xc08288b0 // mova z16.s, p2/M, za1v.s[x12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 5b\n" + "6:" // K loop: Main loop: First: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c22e2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x23, x28]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0x25756141 // dup p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + ".inst 0x257d6140 // dup p0.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2287 // ld1b { za0h.b[x13, #7] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0xc0828890 // mova z16.s, p2/M, za1v.s[x12]\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0828831 // mova z17.s, p2/M, za0v.s[x12, #1]\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xc08288b0 // mova z16.s, p2/M, za1v.s[x12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "sdot z19.s, z17.b, z20.b\n" + "incb x26\n" + "sdot z18.s, z16.b, z20.b\n" + "incb x28\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 8f\n" + "7:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c22e0 // ld1b { za0h.b[x13] }, p0/Z, [x23, x28]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25656141 // dup p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01c22c1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256d6140 // dup p0.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2285 // ld1b { za0h.b[x13, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + ".inst 0xc0828911 // mova z17.s, p2/M, za2v.s[x12]\n" + ".inst 0xc0828990 // mova z16.s, p2/M, za3v.s[x12]\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0828931 // mova z17.s, p2/M, za2v.s[x12, #1]\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xc08289b0 // mova z16.s, p2/M, za3v.s[x12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 7b\n" + "8:" // K loop: Main loop: Second: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c22e0 // ld1b { za0h.b[x13] }, p0/Z, [x23, x28]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25656141 // dup p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01c22c1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256d6140 // dup p0.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2285 // ld1b { za0h.b[x13, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + ".inst 0xc0828911 // mova z17.s, p2/M, za2v.s[x12]\n" + ".inst 0xc0828990 // mova z16.s, p2/M, za3v.s[x12]\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0828931 // mova z17.s, p2/M, za2v.s[x12, #1]\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xc08289b0 // mova z16.s, p2/M, za3v.s[x12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "sdot z19.s, z17.b, z20.b\n" + "incb x26\n" + "sdot z18.s, z16.b, z20.b\n" + "incb x28\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "subs x19, x19, #0x1\n" + "bgt 4b\n" + "9:" // K loop: Tails + "cbnz x11, 12f\n" + "mov x25, %x[in]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: First + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0828890 // mova z16.s, p2/M, za1v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25356141 // dup p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26e2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x23, x28]\n" + "ldr x22, [x25, x16, LSL #0x3]\n" + "addvl x27, x27, #2\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + "add x25, x25, #0x8\n" + "add x13, x13, #0x4\n" + "add x12, x12, #0x1\n" + "cmp x12, x16\n" + "blt 10b\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "11:" // K loop: Tails: Even: Second + ".inst 0xc0828911 // mova z17.s, p2/M, za2v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0828990 // mova z16.s, p2/M, za3v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "add x19, x19, #0x4\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 11b\n" + "whilelt p9.b, x26, %x[width]\n" + "b 14f\n" + "12:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "13:" // K loop: Tails: Odd: Loop + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0828890 // mova z16.s, p2/M, za1v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 13b\n" + "14:" // K loop: End + "st1w { z19.s }, p2, [x27]\n" + "st1w { z18.s }, p2, [x27, #1, MUL VL]\n" + "addvl x27, x27, #2\n" + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p8", "p9", "p10", "p11", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp new file mode 100644 index 0000000000..cd4a76654b --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 4, VLType::SME, false>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntb x20\n" + "cntw x16\n" + "cntw x15, ALL, MUL #2\n" + "cntw x14, ALL, MUL #3\n" + "mov x19, %x[width]\n" + "incb x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x20\n" // n_passes = ceildiv(width, VL<T>) + "sub x13, x19, #0x1\n" + "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x11, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x10, x20, #0x1\n" + "ands x10, x19, x10\n" + "csel x10, x10, x20, NE\n" + "add x10, x10, #0x3\n" + "lsr x10, x10, #0x2\n" + "sub x9, x16, #0x2\n" + "ptrue p11.s\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "lsl x19, x16, #0x1\n" + "whilelt p9.b, XZR, x20\n" + "whilelt p8.b, x19, x20\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "mov x26, #0x0\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "mov x12, #0x0\n" + "cbz x9, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c02e0 // ld1b { za0h.b[x12] }, p0/Z, [x23, x28]\n" + ".inst 0x252c6140 // dup p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25646141 // dup p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0xe01c02c1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256c6140 // dup p0.b, p8.b/Z, p10.b[w12, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c06a4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c0285 // ld1b { za0h.b[x12, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "add x12, x12, #0x8\n" + "cmp x12, x9, LSL #2\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c02e0 // ld1b { za0h.b[x12] }, p0/Z, [x23, x28]\n" + ".inst 0x252c6140 // dup p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25646141 // dup p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0xe01c02c1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256c6140 // dup p0.b, p8.b/Z, p10.b[w12, #5]\n" + "mov x25, %x[in]\n" + ".inst 0xe01c06a4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x21, x28]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c0285 // ld1b { za0h.b[x12, #5] }, p0/Z, [x20, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "incb x28\n" + "incb x26\n" + "cbz x13, 8f\n" + "mov x19, x13\n" + "3:" // K loop: Main loop + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c22e2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x23, x28]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0x25756141 // dup p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + ".inst 0x257d6140 // dup p0.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2287 // ld1b { za0h.b[x13, #7] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c22e2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x23, x28]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0x25756141 // dup p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + ".inst 0x257d6140 // dup p0.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2287 // ld1b { za0h.b[x13, #7] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "incb x26\n" + "incb x28\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c22e0 // ld1b { za0h.b[x13] }, p0/Z, [x23, x28]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25656141 // dup p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01c22c1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256d6140 // dup p0.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2285 // ld1b { za0h.b[x13, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c22e0 // ld1b { za0h.b[x13] }, p0/Z, [x23, x28]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25656141 // dup p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01c22c1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256d6140 // dup p0.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2285 // ld1b { za0h.b[x13, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "incb x26\n" + "incb x28\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "subs x19, x19, #0x1\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x11, 11f\n" + "mov x25, %x[in]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25356141 // dup p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + "addvl x27, x27, #2\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26e2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x23, x28]\n" + "ldr x22, [x25, x16, LSL #0x3]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + "add x25, x25, #0x8\n" + "add x13, x13, #0x4\n" + "add x12, x12, #0x1\n" + "cmp x12, x16\n" + "blt 9b\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "add x19, x19, #0x4\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 10b\n" + "whilelt p9.b, x26, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p8", "p9", "p10", "p11", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp new file mode 100644 index 0000000000..5a71613feb --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 4, VLType::SME, true>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov z20.b, #0x1\n" + "mov z19.s, #0x0\n" + "cntb x20\n" + "mov z18.s, #0x0\n" + "cntw x16\n" + "cntw x15, ALL, MUL #2\n" + "cntw x14, ALL, MUL #3\n" + "ptrue p2.b\n" + "mov x19, %x[width]\n" + "incb x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x20\n" // n_passes = ceildiv(width, VL<T>) + "sub x13, x19, #0x1\n" + "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x11, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x10, x20, #0x1\n" + "ands x10, x19, x10\n" + "csel x10, x10, x20, NE\n" + "add x10, x10, #0x3\n" + "lsr x10, x10, #0x2\n" + "sub x9, x16, #0x2\n" + "ptrue p11.s\n" + "lsl x20, %x[height], #0x1\n" // height * 2 + "lsl x19, x16, #0x1\n" + "whilelt p9.b, XZR, x20\n" + "whilelt p8.b, x19, x20\n" + "zip1 p10.b, p9.b, p8.b\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "mov x26, #0x0\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "cbnz %x[first], 1f\n" + "addvl x27, x27, #-2\n" + "ld1w { z19.s }, p2/Z, [x27]\n" + "ld1w { z18.s }, p2/Z, [x27, #1, MUL VL]\n" + "1:" // K loop: Load row sums: End + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "mov x12, #0x0\n" + "cbz x9, 3f\n" + "2:" // K loop: Charge: Loop + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c02e0 // ld1b { za0h.b[x12] }, p0/Z, [x23, x28]\n" + ".inst 0x252c6140 // dup p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25646141 // dup p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0xe01c02c1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256c6140 // dup p0.b, p8.b/Z, p10.b[w12, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c06a4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c0285 // ld1b { za0h.b[x12, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "add x12, x12, #0x8\n" + "cmp x12, x9, LSL #2\n" + "blt 2b\n" + "3:" // K loop: Charge: End + ".inst 0x25246140 // dup p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c02e0 // ld1b { za0h.b[x12] }, p0/Z, [x23, x28]\n" + ".inst 0x252c6140 // dup p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0x25646141 // dup p1.b, p8.b/Z, p10.b[w12, #4]\n" + ".inst 0xe01c02c1 // ld1b { za0h.b[x12, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256c6140 // dup p0.b, p8.b/Z, p10.b[w12, #5]\n" + "mov x25, %x[in]\n" + ".inst 0xe01c06a4 // ld1b { za0h.b[x12, #4] }, p1/Z, [x21, x28]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c0285 // ld1b { za0h.b[x12, #5] }, p0/Z, [x20, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "incb x28\n" + "incb x26\n" + "cbz x13, 9f\n" + "mov x19, x13\n" + "4:" // K loop: Main loop + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 6f\n" + "5:" // K loop: Main loop: First: Loop + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c22e2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x23, x28]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0x25756141 // dup p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + ".inst 0x257d6140 // dup p0.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2287 // ld1b { za0h.b[x13, #7] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0xc0828890 // mova z16.s, p2/M, za1v.s[x12]\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "udot z19.s, z17.b, z20.b\n" + ".inst 0xc0828831 // mova z17.s, p2/M, za0v.s[x12, #1]\n" + "udot z18.s, z16.b, z20.b\n" + ".inst 0xc08288b0 // mova z16.s, p2/M, za1v.s[x12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "udot z19.s, z17.b, z20.b\n" + "udot z18.s, z16.b, z20.b\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 5b\n" + "6:" // K loop: Main loop: First: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25356140 // dup p0.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe01c22e2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x23, x28]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + ".inst 0x25756141 // dup p1.b, p8.b/Z, p10.b[w13, #6]\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + ".inst 0x257d6140 // dup p0.b, p8.b/Z, p10.b[w13, #7]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a6 // ld1b { za0h.b[x13, #6] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2287 // ld1b { za0h.b[x13, #7] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0xc0828890 // mova z16.s, p2/M, za1v.s[x12]\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "udot z19.s, z17.b, z20.b\n" + ".inst 0xc0828831 // mova z17.s, p2/M, za0v.s[x12, #1]\n" + "udot z18.s, z16.b, z20.b\n" + ".inst 0xc08288b0 // mova z16.s, p2/M, za1v.s[x12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "udot z19.s, z17.b, z20.b\n" + "incb x26\n" + "udot z18.s, z16.b, z20.b\n" + "incb x28\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "cbz x9, 8f\n" + "7:" // K loop: Main loop: Second: Loop + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c22e0 // ld1b { za0h.b[x13] }, p0/Z, [x23, x28]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25656141 // dup p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01c22c1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256d6140 // dup p0.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2285 // ld1b { za0h.b[x13, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + ".inst 0xc0828911 // mova z17.s, p2/M, za2v.s[x12]\n" + ".inst 0xc0828990 // mova z16.s, p2/M, za3v.s[x12]\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "udot z19.s, z17.b, z20.b\n" + ".inst 0xc0828931 // mova z17.s, p2/M, za2v.s[x12, #1]\n" + "udot z18.s, z16.b, z20.b\n" + ".inst 0xc08289b0 // mova z16.s, p2/M, za3v.s[x12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "add x13, x13, #0x8\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "udot z19.s, z17.b, z20.b\n" + "udot z18.s, z16.b, z20.b\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 7b\n" + "8:" // K loop: Main loop: Second: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe01c22e0 // ld1b { za0h.b[x13] }, p0/Z, [x23, x28]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25656141 // dup p1.b, p8.b/Z, p10.b[w13, #4]\n" + ".inst 0xe01c22c1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x22, x28]\n" + ".inst 0x256d6140 // dup p0.b, p8.b/Z, p10.b[w13, #5]\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26a4 // ld1b { za0h.b[x13, #4] }, p1/Z, [x21, x28]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + ".inst 0xe01c2285 // ld1b { za0h.b[x13, #5] }, p0/Z, [x20, x28]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + ".inst 0xc0828911 // mova z17.s, p2/M, za2v.s[x12]\n" + ".inst 0xc0828990 // mova z16.s, p2/M, za3v.s[x12]\n" + "add x24, x24, #0x10\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "udot z19.s, z17.b, z20.b\n" + ".inst 0xc0828931 // mova z17.s, p2/M, za2v.s[x12, #1]\n" + "udot z18.s, z16.b, z20.b\n" + ".inst 0xc08289b0 // mova z16.s, p2/M, za3v.s[x12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25706d21 // dup p1.s, p11.s/Z, p9.s[w12, #1]\n" + ".inst 0x25706d20 // dup p0.s, p11.s/Z, p9.s[w12, #1]\n" + "whilelt p9.b, x26, %x[width]\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "udot z19.s, z17.b, z20.b\n" + "incb x26\n" + "udot z18.s, z16.b, z20.b\n" + "incb x28\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "subs x19, x19, #0x1\n" + "bgt 4b\n" + "9:" // K loop: Tails + "cbnz x11, 12f\n" + "mov x25, %x[in]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x13, #0x0\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: First + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0828890 // mova z16.s, p2/M, za1v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25356141 // dup p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + "udot z19.s, z17.b, z20.b\n" + "udot z18.s, z16.b, z20.b\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe01c26e2 // ld1b { za0h.b[x13, #2] }, p1/Z, [x23, x28]\n" + "ldr x22, [x25, x16, LSL #0x3]\n" + "addvl x27, x27, #2\n" + ".inst 0xe01c22c3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x22, x28]\n" + "add x25, x25, #0x8\n" + "add x13, x13, #0x4\n" + "add x12, x12, #0x1\n" + "cmp x12, x16\n" + "blt 10b\n" + "whilelt p9.b, x26, %x[width]\n" + "whilelt p8.b, x26, %x[width]\n" + "mov x19, #0x0\n" + "mov x12, #0x0\n" + "11:" // K loop: Tails: Even: Second + ".inst 0xc0828911 // mova z17.s, p2/M, za2v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0828990 // mova z16.s, p2/M, za3v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "add x19, x19, #0x4\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "udot z19.s, z17.b, z20.b\n" + "udot z18.s, z16.b, z20.b\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 11b\n" + "whilelt p9.b, x26, %x[width]\n" + "b 14f\n" + "12:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "13:" // K loop: Tails: Odd: Loop + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0xc0828890 // mova z16.s, p2/M, za1v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "udot z19.s, z17.b, z20.b\n" + "udot z18.s, z16.b, z20.b\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 13b\n" + "14:" // K loop: End + "st1w { z19.s }, p2, [x27]\n" + "st1w { z18.s }, p2, [x27, #1, MUL VL]\n" + "addvl x27, x27, #2\n" + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p8", "p9", "p10", "p11", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp new file mode 100644 index 0000000000..3ea616f007 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 1, VLType::SME, false>( + __fp16 * &out, const __fp16 * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cnth x27\n" + "cmp %x[height], x27\n" + "cnth x26\n" + "csel x27, %x[height], x27, LT\n" + "mov x25, #0x0\n" + "ptrue p13.s\n" + "sub x27, x27, #0x1\n" + "whilelt p12.h, XZR, %x[height]\n" + "whilelt p11.h, x26, %x[height]\n" + "mov x24, %x[row_offset]\n" + "mov x23, %x[out]\n" + "whilelt p10.h, x25, %x[width]\n" + "whilelt p9.h, x25, %x[width]\n" + "whilelt p8.h, x25, %x[width]\n" + "1:" // Width loop + "add x22, %x[in], XZR, LSL #3\n" + "add x19, %x[in], x26, LSL #3\n" + "ldr x21, [x22], #0x8\n" + "mov x12, #0x0\n" + "ldr x20, [x19], #0x8\n" + "cbz x27, 3f\n" + "2:" // Loads: Loop + ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe05806a0 // ld1h { za0h.h[x12] }, p1/Z, [x21, x24, LSL #1]\n" + "ldr x21, [x22], #0x8\n" + ".inst 0xe0580288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x24, LSL #1]\n" + "add x12, x12, #0x2\n" + "cmp x12, x27, LSL #1\n" + "ldr x20, [x19], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + "sub x19, %x[width], x25\n" + ".inst 0x25286580 // psel p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe05802a0 // ld1h { za0h.h[x12] }, p0/Z, [x21, x24, LSL #1]\n" + ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" + "cmp x19, x26\n" + ".inst 0xe0580288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x24, LSL #1]\n" + "mov x12, #0x0\n" + "csel x19, x19, x26, LT\n" + "4:" // Stores: Loop + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07f82e0 // st1h { za0v.h[x12] }, p0/Z, [x23, XZR, LSL #1]\n" + ".inst 0x25287540 // psel p0.h, p13.h/Z, p10.h[w12]\n" + ".inst 0xe07a82e8 // st1h { za1v.h[x12] }, p0/Z, [x23, x26, LSL #1]\n" + "add x12, x12, #0x1\n" + "cmp x12, x19\n" + "addvl x23, x23, #4\n" + "blt 4b\n" + "inch x25\n" + "whilelt p10.h, x25, %x[width]\n" + "whilelt p9.h, x25, %x[width]\n" + "whilelt p8.h, x25, %x[width]\n" + "inch x24\n" + "b.any 1b\n" + "mov %x[out], x23\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp new file mode 100644 index 0000000000..d7025420e9 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<2, 1, VLType::SME, false>( + float * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x16\n" + "cntw x15, ALL, MUL #2\n" + "cntw x14, ALL, MUL #3\n" + "mov x19, %x[width]\n" + "incw x19\n" + "sub x19, x19, #0x1\n" + "udiv x19, x19, x16\n" // n_passes = ceildiv(width, VL<T>) + "sub x13, x19, #0x1\n" + "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 + "and x11, x19, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "mov x19, %x[width]\n" + "sub x10, x16, #0x1\n" + "ands x10, x19, x10\n" + "csel x10, x10, x16, NE\n" + "sub x9, x16, #0x2\n" + "ptrue p13.s\n" + "whilelt p12.s, XZR, %x[height]\n" + "whilelt p11.s, x16, %x[height]\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "mov x26, #0x0\n" + "whilelt p10.s, x26, %x[width]\n" + "whilelt p9.s, x26, %x[width]\n" + "whilelt p8.s, x26, %x[width]\n" + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "mov x12, #0x0\n" + "cbz x9, 2f\n" + "1:" // K loop: Charge: Loop + ".inst 0x25306580 // dup p0.s, p9.s/Z, p12.s[w12]\n" + ".inst 0xe09c02e0 // ld1w { za0h.s[x12] }, p0/Z, [x23, x28, LSL #2]\n" + ".inst 0x25306160 // dup p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe09c02c4 // ld1w { za1h.s[x12] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0x25706580 // dup p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0xe09c02a1 // ld1w { za0h.s[x12, #1] }, p0/Z, [x21, x28, LSL #2]\n" + ".inst 0x25706160 // dup p0.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe09c0285 // ld1w { za1h.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 1b\n" + "2:" // K loop: Charge: End + ".inst 0x25306580 // dup p0.s, p9.s/Z, p12.s[w12]\n" + ".inst 0xe09c02e0 // ld1w { za0h.s[x12] }, p0/Z, [x23, x28, LSL #2]\n" + ".inst 0x25306160 // dup p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe09c02c4 // ld1w { za1h.s[x12] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0x25706580 // dup p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0xe09c02a1 // ld1w { za0h.s[x12, #1] }, p0/Z, [x21, x28, LSL #2]\n" + ".inst 0x25706160 // dup p0.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe09c0285 // ld1w { za1h.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "incw x28\n" + "incw x26\n" + "cbz x13, 8f\n" + "mov x19, x13\n" + "3:" // K loop: Main loop + "whilelt p9.s, x26, %x[width]\n" + "whilelt p8.s, x26, %x[width]\n" + "mov x12, #0x0\n" + "cbz x9, 5f\n" + "4:" // K loop: Main loop: First: Loop + ".inst 0x25306580 // dup p0.s, p9.s/Z, p12.s[w12]\n" + ".inst 0xe09c02e8 // ld1w { za2h.s[x12] }, p0/Z, [x23, x28, LSL #2]\n" + ".inst 0x25306160 // dup p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe09c02cc // ld1w { za3h.s[x12] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0x25706580 // dup p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0xe09c02a9 // ld1w { za2h.s[x12, #1] }, p0/Z, [x21, x28, LSL #2]\n" + ".inst 0x25706160 // dup p0.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe09c028d // ld1w { za3h.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 4b\n" + "5:" // K loop: Main loop: First: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25306580 // dup p0.s, p9.s/Z, p12.s[w12]\n" + ".inst 0xe09c02e8 // ld1w { za2h.s[x12] }, p0/Z, [x23, x28, LSL #2]\n" + ".inst 0x25306160 // dup p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe09c02cc // ld1w { za3h.s[x12] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0x25706580 // dup p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0xe09c02a9 // ld1w { za2h.s[x12, #1] }, p0/Z, [x21, x28, LSL #2]\n" + ".inst 0x25706160 // dup p0.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe09c028d // ld1w { za3h.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "whilelt p10.s, x26, %x[width]\n" + ".inst 0xe0af8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "incw x26\n" + "incw x28\n" + ".inst 0xe0ae8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "whilelt p9.s, x26, %x[width]\n" + "whilelt p8.s, x26, %x[width]\n" + "mov x12, #0x0\n" + "cbz x9, 7f\n" + "6:" // K loop: Main loop: Second: Loop + ".inst 0x25306580 // dup p0.s, p9.s/Z, p12.s[w12]\n" + ".inst 0xe09c02e0 // ld1w { za0h.s[x12] }, p0/Z, [x23, x28, LSL #2]\n" + ".inst 0x25306160 // dup p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe09c02c4 // ld1w { za1h.s[x12] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0x25706580 // dup p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0xe09c02a1 // ld1w { za0h.s[x12, #1] }, p0/Z, [x21, x28, LSL #2]\n" + ".inst 0x25706160 // dup p0.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe09c0285 // ld1w { za1h.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "add x12, x12, #0x2\n" + "cmp x12, x9\n" + "blt 6b\n" + "7:" // K loop: Main loop: Second: Tail + "mov x25, %x[in]\n" + "add x24, %x[in], x16, LSL #3\n" + ".inst 0x25306580 // dup p0.s, p9.s/Z, p12.s[w12]\n" + ".inst 0xe09c02e0 // ld1w { za0h.s[x12] }, p0/Z, [x23, x28, LSL #2]\n" + ".inst 0x25306160 // dup p0.s, p8.s/Z, p11.s[w12]\n" + ".inst 0xe09c02c4 // ld1w { za1h.s[x12] }, p0/Z, [x22, x28, LSL #2]\n" + ".inst 0x25706580 // dup p0.s, p9.s/Z, p12.s[w12, #1]\n" + ".inst 0xe09c02a1 // ld1w { za0h.s[x12, #1] }, p0/Z, [x21, x28, LSL #2]\n" + ".inst 0x25706160 // dup p0.s, p8.s/Z, p11.s[w12, #1]\n" + ".inst 0xe09c0285 // ld1w { za1h.s[x12, #1] }, p0/Z, [x20, x28, LSL #2]\n" + "ldr x23, [x25, #0x0]\n" + "ldr x22, [x24, #0x0]\n" + "ldr x21, [x25, #0x8]\n" + "ldr x20, [x24, #0x8]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25707541 // dup p1.s, p13.s/Z, p10.s[w12, #1]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25707540 // dup p0.s, p13.s/Z, p10.s[w12, #1]\n" + "whilelt p10.s, x26, %x[width]\n" + ".inst 0xe0af8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x15, LSL #2]\n" + "incw x26\n" + "incw x28\n" + ".inst 0xe0ae836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" + "addvl x27, x27, #4\n" + "subs x19, x19, #0x1\n" + "bgt 3b\n" + "8:" // K loop: Tails + "cbnz x11, 11f\n" + "mov x25, %x[in]\n" + "whilelt p9.s, x26, %x[width]\n" + "whilelt p8.s, x26, %x[width]\n" + "mov x12, #0x0\n" + "9:" // K loop: Tails: Even: First + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25306581 // dup p1.s, p9.s/Z, p12.s[w12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + ".inst 0x25306160 // dup p0.s, p8.s/Z, p11.s[w12]\n" + "addvl x27, x27, #2\n" + "ldr x23, [x25, #0x0]\n" + ".inst 0xe09c06e8 // ld1w { za2h.s[x12] }, p1/Z, [x23, x28, LSL #2]\n" + "ldr x22, [x25, x16, LSL #0x3]\n" + ".inst 0xe09c02cc // ld1w { za3h.s[x12] }, p0/Z, [x22, x28, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x16\n" + "add x25, x25, #0x8\n" + "blt 9b\n" + "whilelt p10.s, x26, %x[width]\n" + "whilelt p9.s, x26, %x[width]\n" + "whilelt p8.s, x26, %x[width]\n" + "mov x12, #0x0\n" + "10:" // K loop: Tails: Even: Second + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 10b\n" + "whilelt p10.s, x26, %x[width]\n" + "b 13f\n" + "11:" // K loop: Tails: Odd + "mov x12, #0x0\n" + "12:" // K loop: Tails: Odd: Loop + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" + "addvl x27, x27, #2\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 12b\n" + "13:" // K loop: End + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p8", "p9", "p10", "p11", "p12", "p13", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp new file mode 100644 index 0000000000..556d1481de --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<4, 2, VLType::SME, false>( + bfloat16 * &out, const bfloat16 * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x15\n" + "cntw x14, ALL, MUL #2\n" + "cntw x13, ALL, MUL #3\n" + "cnth x11\n" + "ptrue p13.s\n" + "cntw x10\n" + "cmp %x[height], x10\n" + "csel x10, %x[height], x10, LT\n" + "sub x10, x10, #0x1\n" + "whilelt p10.h, XZR, %x[height]\n" + "whilelt p9.h, x15, %x[height]\n" + "whilelt p8.h, x14, %x[height]\n" + "zip1 p12.h, p10.h, p8.h\n" + "whilelt p8.h, x13, %x[height]\n" + "zip1 p11.h, p9.h, p8.h\n" + "mov x9, %x[row_offset]\n" + "mov x28, %x[out]\n" + "mov x27, #0x0\n" + "whilelt p10.h, x27, %x[width]\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "1:" // Width loop + "mov x12, #0x0\n" + "add x26, %x[in], XZR, LSL #3\n" + "add x25, %x[in], x15, LSL #3\n" + "add x24, %x[in], x14, LSL #3\n" + "add x23, %x[in], x13, LSL #3\n" + "ldr x22, [x26], #0x8\n" + "ldr x21, [x25], #0x8\n" + "ldr x20, [x24], #0x8\n" + "ldr x19, [x23], #0x8\n" + "cbz x10, 3f\n" + "2:" // Loads: Loop + ".inst 0x25286580 // dup p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe04902c0 // ld1h { za0h.h[x12] }, p0/Z, [x22, x9, LSL #1]\n" + ".inst 0x25286160 // dup p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe04902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x9, LSL #1]\n" + ".inst 0x25386580 // dup p0.h, p9.h/Z, p12.h[w12, #1]\n" + ".inst 0xe0490281 // ld1h { za0h.h[x12, #1] }, p0/Z, [x20, x9, LSL #1]\n" + ".inst 0x25386160 // dup p0.h, p8.h/Z, p11.h[w12, #1]\n" + ".inst 0xe0490269 // ld1h { za1h.h[x12, #1] }, p0/Z, [x19, x9, LSL #1]\n" + "ldr x22, [x26], #0x8\n" + "ldr x21, [x25], #0x8\n" + "ldr x20, [x24], #0x8\n" + "ldr x19, [x23], #0x8\n" + "add x12, x12, #0x2\n" + "cmp x12, x10, LSL #1\n" + "blt 2b\n" + "3:" // Loads: Tail + ".inst 0x25286580 // dup p0.h, p9.h/Z, p12.h[w12]\n" + ".inst 0xe04902c0 // ld1h { za0h.h[x12] }, p0/Z, [x22, x9, LSL #1]\n" + ".inst 0x25286160 // dup p0.h, p8.h/Z, p11.h[w12]\n" + ".inst 0xe04902a8 // ld1h { za1h.h[x12] }, p0/Z, [x21, x9, LSL #1]\n" + ".inst 0x25386580 // dup p0.h, p9.h/Z, p12.h[w12, #1]\n" + ".inst 0xe0490281 // ld1h { za0h.h[x12, #1] }, p0/Z, [x20, x9, LSL #1]\n" + ".inst 0x25386160 // dup p0.h, p8.h/Z, p11.h[w12, #1]\n" + ".inst 0xe0490269 // ld1h { za1h.h[x12, #1] }, p0/Z, [x19, x9, LSL #1]\n" + "mov x12, #0x0\n" + "sub x19, %x[width], x27\n" + "cmp x19, x11\n" + "csel x19, x19, x11, LT\n" + "add x19, x19, #0x1\n" + "lsr x19, x19, #0x1\n" + "4:" // Stores: Loop + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0x25307541 // dup p1.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0af8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x15, LSL #2]\n" + ".inst 0x25307540 // dup p0.s, p13.s/Z, p10.s[w12]\n" + ".inst 0xe0ae8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x14, LSL #2]\n" + ".inst 0xe0ad838c // st1w { za3v.s[x12] }, p0/Z, [x28, x13, LSL #2]\n" + "addvl x28, x28, #4\n" + "add x12, x12, #0x1\n" + "cmp x12, x19\n" + "blt 4b\n" + "inch x9\n" + "inch x27\n" + "whilelt p10.h, x27, %x[width]\n" + "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" + "b.any 1b\n" + "mov %x[out], x28\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p8", "p9", "p10", "p11", "p12", "p13", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp new file mode 100644 index 0000000000..49d1aa5cc5 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<4, 4, VLType::SME, false>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x15\n" + "cntw x14\n" + "cntw x13, ALL, MUL #2\n" + "cntw x11, ALL, MUL #3\n" + "cmp %x[height], x15\n" + "csel x15, %x[height], x15, LT\n" + "whilelt p12.b, XZR, %x[height]\n" + "whilelt p10.b, x14, %x[height]\n" + "whilelt p9.b, x13, %x[height]\n" + "whilelt p8.b, x11, %x[height]\n" + "zip1 p12.b, p12.b, p9.b\n" + "zip1 p10.b, p10.b, p8.b\n" + "mov x10, #0x0\n" + "cntb x9\n" + "ptrue p11.s\n" + "sub x15, x15, #0x1\n" + "zip1 p10.b, p12.b, p10.b\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "whilelt p9.b, x10, %x[width]\n" + "whilelt p8.b, x10, %x[width]\n" + "1:" // Width loop + "add x26, %x[in], XZR, LSL #3\n" + "add x25, %x[in], x14, LSL #3\n" + "ldr x24, [x26], #0x8\n" + "add x23, %x[in], x13, LSL #3\n" + "add x22, %x[in], x11, LSL #3\n" + "ldr x19, [x25], #0x8\n" + "mov x12, #0x0\n" + "ldr x21, [x23], #0x8\n" + "ldr x20, [x22], #0x8\n" + "cbz x15, 3f\n" + "2:" // Loads: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x24, [x26], #0x8\n" + ".inst 0xe01c0261 // ld1b { za0h.b[x12, #1] }, p0/Z, [x19, x28]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x19, [x25], #0x8\n" + ".inst 0xe01c06a2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x21, x28]\n" + "ldr x21, [x23], #0x8\n" + ".inst 0xe01c0283 // ld1b { za0h.b[x12, #3] }, p0/Z, [x20, x28]\n" + "add x12, x12, #0x4\n" + "cmp x12, x15, LSL #2\n" + "ldr x20, [x22], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe01c0261 // ld1b { za0h.b[x12, #1] }, p0/Z, [x19, x28]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" + "sub x19, %x[width], x10\n" + ".inst 0xe01c02a2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x21, x28]\n" + "cmp x19, x9\n" + "csel x19, x19, x9, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "add x19, x19, #0x3\n" + ".inst 0xe01c0283 // ld1b { za0h.b[x12, #3] }, p0/Z, [x20, x28]\n" + "mov x12, #0x0\n" + "lsr x19, x19, #0x2\n" + "4:" // Stores: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0ae8364 // st1w { za1v.s[x12] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0ad8768 // st1w { za2v.s[x12] }, p1/Z, [x27, x13, LSL #2]\n" + ".inst 0xe0ab836c // st1w { za3v.s[x12] }, p0/Z, [x27, x11, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x19\n" + "addvl x27, x27, #4\n" + "blt 4b\n" + "incb x10\n" + "whilelt p9.b, x10, %x[width]\n" + "whilelt p8.b, x10, %x[width]\n" + "incb x28\n" + "b.any 1b\n" + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp new file mode 100644 index 0000000000..94673d41d8 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<4, 4, VLType::SME, true>( + int8_t * &out, const int8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov z24.b, #0x1\n" + "mov z23.s, #0x0\n" + "ptrue p2.b\n" + "mov z22.s, #0x0\n" + "cntw x16\n" + "mov z21.s, #0x0\n" + "cntw x15, ALL, MUL #2\n" + "mov z20.s, #0x0\n" + "cntw x14, ALL, MUL #3\n" + "cntb x11\n" + "ptrue p11.s\n" + "cntw x10\n" + "cmp %x[height], x10\n" + "csel x10, %x[height], x10, LT\n" + "sub x10, x10, #0x1\n" + "whilelt p10.b, XZR, %x[height]\n" + "whilelt p9.b, x16, %x[height]\n" + "whilelt p8.b, x15, %x[height]\n" + "zip1 p10.b, p10.b, p8.b\n" + "whilelt p8.b, x14, %x[height]\n" + "zip1 p9.b, p9.b, p8.b\n" + "mov x9, %x[row_offset]\n" + "mov x28, %x[out]\n" + "zip1 p10.b, p10.b, p9.b\n" + "cbnz %x[first], 1f\n" + "addvl x28, x28, #-4\n" + "ld1w { z23.s }, p2/Z, [x28]\n" + "ld1w { z22.s }, p2/Z, [x28, #1, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x28, #2, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x28, #3, MUL VL]\n" + "1:" // Initialise row sums: End + "mov x27, #0x0\n" + "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" + "2:" // Width loop + "mov x13, #0x0\n" + "add x26, %x[in], XZR, LSL #3\n" + "add x25, %x[in], x16, LSL #3\n" + "add x24, %x[in], x15, LSL #3\n" + "add x23, %x[in], x14, LSL #3\n" + "ldr x22, [x26], #0x8\n" + "ldr x21, [x25], #0x8\n" + "ldr x19, [x24], #0x8\n" + "ldr x20, [x23], #0x8\n" + "cbz x10, 4f\n" + "3:" // Loads: Loop + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe00922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x9]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25356141 // dup p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe00922a1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x21, x9]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + "ldr x22, [x26], #0x8\n" + ".inst 0xe0092662 // ld1b { za0h.b[x13, #2] }, p1/Z, [x19, x9]\n" + "ldr x21, [x25], #0x8\n" + "ldr x19, [x24], #0x8\n" + ".inst 0xe0092283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x9]\n" + "ldr x20, [x23], #0x8\n" + "add x13, x13, #0x4\n" + "cmp x13, x10, LSL #2\n" + "blt 3b\n" + "4:" // Loads: Tail + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe00922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x9]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25356141 // dup p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe00922a1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x21, x9]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + "mov x12, #0x0\n" + ".inst 0xe0092662 // ld1b { za0h.b[x13, #2] }, p1/Z, [x19, x9]\n" + "sub x19, %x[width], x27\n" + "cmp x19, x11\n" + ".inst 0xe0092283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x9]\n" + "csel x19, x19, x11, LT\n" + "add x19, x19, #0x3\n" + "lsr x19, x19, #0x2\n" + "5:" // Stores: Loop + ".inst 0xc0828813 // mova z19.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" + ".inst 0xc0828892 // mova z18.s, p2/M, za1v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828911 // mova z17.s, p2/M, za2v.s[x12]\n" + ".inst 0xe0b08384 // st1w { za1v.s[x12] }, p0/Z, [x28, x16, LSL #2]\n" + ".inst 0x25306d21 // dup p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828990 // mova z16.s, p2/M, za3v.s[x12]\n" + "sdot z23.s, z19.b, z24.b\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "sdot z22.s, z18.b, z24.b\n" + ".inst 0xe0af8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x15, LSL #2]\n" + "sdot z21.s, z17.b, z24.b\n" + "sdot z20.s, z16.b, z24.b\n" + ".inst 0xe0ae838c // st1w { za3v.s[x12] }, p0/Z, [x28, x14, LSL #2]\n" + "addvl x28, x28, #4\n" + "add x12, x12, #0x1\n" + "cmp x12, x19\n" + "blt 5b\n" + "incb x9\n" + "incb x27\n" + "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" + "b.any 2b\n" + "st1w { z23.s }, p2, [x28]\n" + "st1w { z22.s }, p2, [x28, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x28, #2, MUL VL]\n" + "st1w { z20.s }, p2, [x28, #3, MUL VL]\n" + "addvl x28, x28, #4\n" + "mov %x[out], x28\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p8", "p9", "p10", "p11", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp new file mode 100644 index 0000000000..bbdaaa3217 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<4, 4, VLType::SME, false>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x15\n" + "cntw x14\n" + "cntw x13, ALL, MUL #2\n" + "cntw x11, ALL, MUL #3\n" + "cmp %x[height], x15\n" + "csel x15, %x[height], x15, LT\n" + "whilelt p12.b, XZR, %x[height]\n" + "whilelt p10.b, x14, %x[height]\n" + "whilelt p9.b, x13, %x[height]\n" + "whilelt p8.b, x11, %x[height]\n" + "zip1 p12.b, p12.b, p9.b\n" + "zip1 p10.b, p10.b, p8.b\n" + "mov x10, #0x0\n" + "cntb x9\n" + "ptrue p11.s\n" + "sub x15, x15, #0x1\n" + "zip1 p10.b, p12.b, p10.b\n" + "mov x28, %x[row_offset]\n" + "mov x27, %x[out]\n" + "whilelt p9.b, x10, %x[width]\n" + "whilelt p8.b, x10, %x[width]\n" + "1:" // Width loop + "add x26, %x[in], XZR, LSL #3\n" + "add x25, %x[in], x14, LSL #3\n" + "ldr x24, [x26], #0x8\n" + "add x23, %x[in], x13, LSL #3\n" + "add x22, %x[in], x11, LSL #3\n" + "ldr x19, [x25], #0x8\n" + "mov x12, #0x0\n" + "ldr x21, [x23], #0x8\n" + "ldr x20, [x22], #0x8\n" + "cbz x15, 3f\n" + "2:" // Loads: Loop + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + "ldr x24, [x26], #0x8\n" + ".inst 0xe01c0261 // ld1b { za0h.b[x12, #1] }, p0/Z, [x19, x28]\n" + ".inst 0x25346141 // psel p1.b, p8.b/Z, p10.b[w12, #2]\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "ldr x19, [x25], #0x8\n" + ".inst 0xe01c06a2 // ld1b { za0h.b[x12, #2] }, p1/Z, [x21, x28]\n" + "ldr x21, [x23], #0x8\n" + ".inst 0xe01c0283 // ld1b { za0h.b[x12, #3] }, p0/Z, [x20, x28]\n" + "add x12, x12, #0x4\n" + "cmp x12, x15, LSL #2\n" + "ldr x20, [x22], #0x8\n" + "blt 2b\n" + "3:" // Loads: Tail + ".inst 0x25246140 // psel p0.b, p8.b/Z, p10.b[w12]\n" + ".inst 0xe01c0300 // ld1b { za0h.b[x12] }, p0/Z, [x24, x28]\n" + ".inst 0x252c6140 // psel p0.b, p8.b/Z, p10.b[w12, #1]\n" + ".inst 0xe01c0261 // ld1b { za0h.b[x12, #1] }, p0/Z, [x19, x28]\n" + ".inst 0x25346140 // psel p0.b, p8.b/Z, p10.b[w12, #2]\n" + "sub x19, %x[width], x10\n" + ".inst 0xe01c02a2 // ld1b { za0h.b[x12, #2] }, p0/Z, [x21, x28]\n" + "cmp x19, x9\n" + "csel x19, x19, x9, LT\n" + ".inst 0x253c6140 // psel p0.b, p8.b/Z, p10.b[w12, #3]\n" + "add x19, x19, #0x3\n" + ".inst 0xe01c0283 // ld1b { za0h.b[x12, #3] }, p0/Z, [x20, x28]\n" + "mov x12, #0x0\n" + "lsr x19, x19, #0x2\n" + "4:" // Stores: Loop + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0ae8364 // st1w { za1v.s[x12] }, p0/Z, [x27, x14, LSL #2]\n" + ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0ad8768 // st1w { za2v.s[x12] }, p1/Z, [x27, x13, LSL #2]\n" + ".inst 0xe0ab836c // st1w { za3v.s[x12] }, p0/Z, [x27, x11, LSL #2]\n" + "add x12, x12, #0x1\n" + "cmp x12, x19\n" + "addvl x27, x27, #4\n" + "blt 4b\n" + "incb x10\n" + "whilelt p9.b, x10, %x[width]\n" + "whilelt p8.b, x10, %x[width]\n" + "incb x28\n" + "b.any 1b\n" + "mov %x[out], x27\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp new file mode 100644 index 0000000000..961008a3f2 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<4, 4, VLType::SME, true>( + uint8_t * &out, const uint8_t * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "mov z24.b, #0x1\n" + "mov z23.s, #0x0\n" + "ptrue p2.b\n" + "mov z22.s, #0x0\n" + "cntw x16\n" + "mov z21.s, #0x0\n" + "cntw x15, ALL, MUL #2\n" + "mov z20.s, #0x0\n" + "cntw x14, ALL, MUL #3\n" + "cntb x11\n" + "ptrue p11.s\n" + "cntw x10\n" + "cmp %x[height], x10\n" + "csel x10, %x[height], x10, LT\n" + "sub x10, x10, #0x1\n" + "whilelt p10.b, XZR, %x[height]\n" + "whilelt p9.b, x16, %x[height]\n" + "whilelt p8.b, x15, %x[height]\n" + "zip1 p10.b, p10.b, p8.b\n" + "whilelt p8.b, x14, %x[height]\n" + "zip1 p9.b, p9.b, p8.b\n" + "mov x9, %x[row_offset]\n" + "mov x28, %x[out]\n" + "zip1 p10.b, p10.b, p9.b\n" + "cbnz %x[first], 1f\n" + "addvl x28, x28, #-4\n" + "ld1w { z23.s }, p2/Z, [x28]\n" + "ld1w { z22.s }, p2/Z, [x28, #1, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x28, #2, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x28, #3, MUL VL]\n" + "1:" // Initialise row sums: End + "mov x27, #0x0\n" + "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" + "2:" // Width loop + "mov x13, #0x0\n" + "add x26, %x[in], XZR, LSL #3\n" + "add x25, %x[in], x16, LSL #3\n" + "add x24, %x[in], x15, LSL #3\n" + "add x23, %x[in], x14, LSL #3\n" + "ldr x22, [x26], #0x8\n" + "ldr x21, [x25], #0x8\n" + "ldr x19, [x24], #0x8\n" + "ldr x20, [x23], #0x8\n" + "cbz x10, 4f\n" + "3:" // Loads: Loop + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe00922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x9]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25356141 // dup p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe00922a1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x21, x9]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + "ldr x22, [x26], #0x8\n" + ".inst 0xe0092662 // ld1b { za0h.b[x13, #2] }, p1/Z, [x19, x9]\n" + "ldr x21, [x25], #0x8\n" + "ldr x19, [x24], #0x8\n" + ".inst 0xe0092283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x9]\n" + "ldr x20, [x23], #0x8\n" + "add x13, x13, #0x4\n" + "cmp x13, x10, LSL #2\n" + "blt 3b\n" + "4:" // Loads: Tail + ".inst 0x25256140 // dup p0.b, p8.b/Z, p10.b[w13]\n" + ".inst 0xe00922c0 // ld1b { za0h.b[x13] }, p0/Z, [x22, x9]\n" + ".inst 0x252d6140 // dup p0.b, p8.b/Z, p10.b[w13, #1]\n" + ".inst 0x25356141 // dup p1.b, p8.b/Z, p10.b[w13, #2]\n" + ".inst 0xe00922a1 // ld1b { za0h.b[x13, #1] }, p0/Z, [x21, x9]\n" + ".inst 0x253d6140 // dup p0.b, p8.b/Z, p10.b[w13, #3]\n" + "mov x12, #0x0\n" + ".inst 0xe0092662 // ld1b { za0h.b[x13, #2] }, p1/Z, [x19, x9]\n" + "sub x19, %x[width], x27\n" + "cmp x19, x11\n" + ".inst 0xe0092283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x9]\n" + "csel x19, x19, x11, LT\n" + "add x19, x19, #0x3\n" + "lsr x19, x19, #0x2\n" + "5:" // Stores: Loop + ".inst 0xc0828813 // mova z19.s, p2/M, za0v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" + ".inst 0xc0828892 // mova z18.s, p2/M, za1v.s[x12]\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828911 // mova z17.s, p2/M, za2v.s[x12]\n" + ".inst 0xe0b08384 // st1w { za1v.s[x12] }, p0/Z, [x28, x16, LSL #2]\n" + ".inst 0x25306d21 // dup p1.s, p11.s/Z, p9.s[w12]\n" + ".inst 0xc0828990 // mova z16.s, p2/M, za3v.s[x12]\n" + "udot z23.s, z19.b, z24.b\n" + ".inst 0x25306d20 // dup p0.s, p11.s/Z, p9.s[w12]\n" + "udot z22.s, z18.b, z24.b\n" + ".inst 0xe0af8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x15, LSL #2]\n" + "udot z21.s, z17.b, z24.b\n" + "udot z20.s, z16.b, z24.b\n" + ".inst 0xe0ae838c // st1w { za3v.s[x12] }, p0/Z, [x28, x14, LSL #2]\n" + "addvl x28, x28, #4\n" + "add x12, x12, #0x1\n" + "cmp x12, x19\n" + "blt 5b\n" + "incb x9\n" + "incb x27\n" + "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" + "b.any 2b\n" + "st1w { z23.s }, p2, [x28]\n" + "st1w { z22.s }, p2, [x28, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x28, #2, MUL VL]\n" + "st1w { z20.s }, p2, [x28, #3, MUL VL]\n" + "addvl x28, x28, #4\n" + "mov %x[out], x28\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p8", "p9", "p10", "p11", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp new file mode 100644 index 0000000000..141ab00a52 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(__ARM_FEATURE_SVE) + +template <> +void interleave_block<4, 1, VLType::SME, false>( + float * &out, const float * const *in, + size_t width, size_t height, size_t row_offset, bool first +) +{ + ARM_COMPUTE_UNUSED(first); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cntw x14\n" + "cntw x13, ALL, MUL #2\n" + "cntw x11, ALL, MUL #3\n" + "ptrue p3.s\n" + "cntw x10\n" + "cmp %x[height], x10\n" + "csel x10, %x[height], x10, LT\n" + "sub x10, x10, #0x1\n" + "whilelt p2.s, XZR, %x[height]\n" + "whilelt p15.s, x14, %x[height]\n" + "whilelt p14.s, x13, %x[height]\n" + "whilelt p13.s, x11, %x[height]\n" + "mov x9, %x[row_offset]\n" + "mov x28, %x[out]\n" + "mov x27, #0x0\n" + "whilelt p12.s, x27, %x[width]\n" + "whilelt p11.s, x27, %x[width]\n" + "whilelt p10.s, x27, %x[width]\n" + "whilelt p9.s, x27, %x[width]\n" + "whilelt p8.s, x27, %x[width]\n" + "1:" // Width loop + "mov x12, #0x0\n" + "add x26, %x[in], XZR, LSL #3\n" + "add x25, %x[in], x14, LSL #3\n" + "add x24, %x[in], x13, LSL #3\n" + "add x23, %x[in], x11, LSL #3\n" + "ldr x22, [x26], #0x8\n" + "ldr x21, [x25], #0x8\n" + "ldr x20, [x24], #0x8\n" + "ldr x19, [x23], #0x8\n" + "cbz x10, 3f\n" + "2:" // Loads: Loop + ".inst 0x25306c40 // dup p0.s, p11.s/Z, p2.s[w12]\n" + ".inst 0xe08902c0 // ld1w { za0h.s[x12] }, p0/Z, [x22, x9, LSL #2]\n" + ".inst 0x253069e0 // dup p0.s, p10.s/Z, p15.s[w12]\n" + ".inst 0xe08902a4 // ld1w { za1h.s[x12] }, p0/Z, [x21, x9, LSL #2]\n" + ".inst 0x253065c0 // dup p0.s, p9.s/Z, p14.s[w12]\n" + ".inst 0xe0890288 // ld1w { za2h.s[x12] }, p0/Z, [x20, x9, LSL #2]\n" + ".inst 0x253061a0 // dup p0.s, p8.s/Z, p13.s[w12]\n" + ".inst 0xe089026c // ld1w { za3h.s[x12] }, p0/Z, [x19, x9, LSL #2]\n" + "ldr x22, [x26], #0x8\n" + "ldr x21, [x25], #0x8\n" + "ldr x20, [x24], #0x8\n" + "ldr x19, [x23], #0x8\n" + "add x12, x12, #0x1\n" + "cmp x12, x10\n" + "blt 2b\n" + "3:" // Loads: Tail + ".inst 0x25306c40 // dup p0.s, p11.s/Z, p2.s[w12]\n" + ".inst 0xe08902c0 // ld1w { za0h.s[x12] }, p0/Z, [x22, x9, LSL #2]\n" + ".inst 0x253069e0 // dup p0.s, p10.s/Z, p15.s[w12]\n" + ".inst 0xe08902a4 // ld1w { za1h.s[x12] }, p0/Z, [x21, x9, LSL #2]\n" + ".inst 0x253065c0 // dup p0.s, p9.s/Z, p14.s[w12]\n" + ".inst 0xe0890288 // ld1w { za2h.s[x12] }, p0/Z, [x20, x9, LSL #2]\n" + ".inst 0x253061a0 // dup p0.s, p8.s/Z, p13.s[w12]\n" + ".inst 0xe089026c // ld1w { za3h.s[x12] }, p0/Z, [x19, x9, LSL #2]\n" + "mov x12, #0x0\n" + "sub x19, %x[width], x27\n" + "cmp x19, x14\n" + "csel x19, x19, x14, LT\n" + "4:" // Stores: Loop + ".inst 0x25304d80 // dup p0.s, p3.s/Z, p12.s[w12]\n" + ".inst 0xe0bf8380 // st1w { za0v.s[x12] }, p0/Z, [x28, XZR, LSL #2]\n" + ".inst 0x25304d80 // dup p0.s, p3.s/Z, p12.s[w12]\n" + ".inst 0x25304d81 // dup p1.s, p3.s/Z, p12.s[w12]\n" + ".inst 0xe0ae8384 // st1w { za1v.s[x12] }, p0/Z, [x28, x14, LSL #2]\n" + ".inst 0x25304d80 // dup p0.s, p3.s/Z, p12.s[w12]\n" + ".inst 0xe0ad8788 // st1w { za2v.s[x12] }, p1/Z, [x28, x13, LSL #2]\n" + ".inst 0xe0ab838c // st1w { za3v.s[x12] }, p0/Z, [x28, x11, LSL #2]\n" + "addvl x28, x28, #4\n" + "add x12, x12, #0x1\n" + "cmp x12, x19\n" + "blt 4b\n" + "incw x9\n" + "incw x27\n" + "whilelt p12.s, x27, %x[width]\n" + "whilelt p11.s, x27, %x[width]\n" + "whilelt p10.s, x27, %x[width]\n" + "whilelt p9.s, x27, %x[width]\n" + "whilelt p8.s, x27, %x[width]\n" + "b.any 1b\n" + "mov %x[out], x28\n" + ".inst 0xd503467f // SMSTOP\n" + : [out] "+&r" (out) + : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +#endif // defined(__ARM_FEATURE_SVE) |