From fc94f4d23abd4bc427b701f54ad85282e9ec7872 Mon Sep 17 00:00:00 2001 From: Michael Tyler Date: Tue, 4 Jun 2024 15:47:37 +0100 Subject: Update CPU kernels and add mixed sign GEMM support - Add support for mixed sign quantized convolution. - Add support for mixed sign dequantized GEMM. - Add SME FP16 GEMV kernel. - Change SME vector length function to use RDSVL instead of static variable. - Add GEMM dilation support internally (not exposed yet). - Remove unused "get_default_activation_values" functions. - Add SVE fixed format interleaved BF16 DOT kernel. - Updates and optimizations to assembly kernels. Resolves COMPMID-6926 Change-Id: I227f502502611d4cc4111c89e30c53ce94079544 Signed-off-by: Michael Tyler Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/11570 Tested-by: Arm Jenkins Reviewed-by: Gunes Bayir Comments-Addressed: Arm Jenkins Benchmark: Arm Jenkins --- .../sme_transpose_interleave_16VL_2x2_fp32bf16.hpp | 158 ++++++++++----------- 1 file changed, 79 insertions(+), 79 deletions(-) (limited to 'src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp') diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp index 2756327815..896288cdda 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023 Arm Limited. + * Copyright (c) 2022-2024 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,133 +43,133 @@ void sme_transpose_interleave_16VL_2x2_fp32bf16(bfloat16 *out, const float *in, "ptrue p7.b\n" "1:" // Main row loop: Head "mov x25, %x[in]\n" - "add x24, x25, %x[in_stride]\n" "cmp %x[height], #0x1\n" - "add %x[in], x24, %x[in_stride]\n" + "add x24, x25, %x[in_stride]\n" "mov x23, %x[out]\n" + "add %x[in], x24, %x[in_stride]\n" "csel x24, x24, %x[pad_row], GT\n" "sub %x[height], %x[height], #0x2\n" "mov x22, %x[width]\n" "2:" // Main row loop: Column loop "mov x21, x22\n" + "mov x20, x23\n" "whilelt p1.s, XZR, x21\n" - "ld1w { z16.s }, p1/Z, [x25]\n" - ".inst 0x658abe00 // bfcvt z0.h, p7/M, z16.s\n" "decw x21\n" "whilelt p0.s, XZR, x21\n" - "ld1w { z16.s }, p0/Z, [x25, #1, MUL VL]\n" - ".inst 0x658abe1f // bfcvt z31.h, p7/M, z16.s\n" "decw x21\n" + "ld1w { z16.s }, p1/Z, [x25]\n" "whilelt p6.s, XZR, x21\n" - "ld1w { z16.s }, p6/Z, [x25, #2, MUL VL]\n" - ".inst 0x658abe1e // bfcvt z30.h, p7/M, z16.s\n" "decw x21\n" + "ld1w { z18.s }, p0/Z, [x25, #1, MUL VL]\n" "whilelt p5.s, XZR, x21\n" - "ld1w { z16.s }, p5/Z, [x25, #3, MUL VL]\n" - ".inst 0x658abe1d // bfcvt z29.h, p7/M, z16.s\n" "decw x21\n" + "ld1w { z17.s }, p6/Z, [x25, #2, MUL VL]\n" "whilelt p4.s, XZR, x21\n" - "ld1w { z16.s }, p4/Z, [x25, #4, MUL VL]\n" - ".inst 0x658abe1c // bfcvt z28.h, p7/M, z16.s\n" "decw x21\n" + "ld1w { z19.s }, p5/Z, [x25, #3, MUL VL]\n" + ".inst 0x658abe03 // bfcvt z3.h, p7/M, z16.s\n" "whilelt p3.s, XZR, x21\n" - "ld1w { z16.s }, p3/Z, [x25, #5, MUL VL]\n" - ".inst 0x658abe1b // bfcvt z27.h, p7/M, z16.s\n" "decw x21\n" + "ld1w { z16.s }, p4/Z, [x25, #4, MUL VL]\n" + ".inst 0x658abe42 // bfcvt z2.h, p7/M, z18.s\n" "whilelt p2.s, XZR, x21\n" - "ld1w { z16.s }, p2/Z, [x25, #6, MUL VL]\n" - ".inst 0x658abe1a // bfcvt z26.h, p7/M, z16.s\n" "decw x21\n" - "ld1w { z16.s }, p1/Z, [x24]\n" + "ld1w { z18.s }, p3/Z, [x25, #5, MUL VL]\n" + ".inst 0x658abe21 // bfcvt z1.h, p7/M, z17.s\n" + "ld1w { z17.s }, p2/Z, [x25, #6, MUL VL]\n" + ".inst 0x658abe60 // bfcvt z0.h, p7/M, z19.s\n" + "decw x22, ALL, MUL #16\n" + "add x23, x23, %x[out_stride]\n" + ".inst 0x658abe1f // bfcvt z31.h, p7/M, z16.s\n" + "ld1w { z19.s }, p1/Z, [x24]\n" "whilelt p1.s, XZR, x21\n" - ".inst 0x648abe00 // bfcvtnt z0.h, p7/M, z16.s\n" "decw x21\n" "ld1w { z16.s }, p1/Z, [x25, #7, MUL VL]\n" "addvl x25, x25, #16\n" - ".inst 0x658abe19 // bfcvt z25.h, p7/M, z16.s\n" - "ld1w { z16.s }, p0/Z, [x24, #1, MUL VL]\n" + ".inst 0x658abe5e // bfcvt z30.h, p7/M, z18.s\n" + ".inst 0x658abe3d // bfcvt z29.h, p7/M, z17.s\n" + "ld1w { z18.s }, p0/Z, [x24, #1, MUL VL]\n" "whilelt p0.s, XZR, x21\n" "decw x21\n" - ".inst 0x648abe1f // bfcvtnt z31.h, p7/M, z16.s\n" - "ld1w { z16.s }, p0/Z, [x25, #-8, MUL VL]\n" - ".inst 0x658abe18 // bfcvt z24.h, p7/M, z16.s\n" - "mov x20, x23\n" - "decw x22, ALL, MUL #16\n" - "ld1w { z16.s }, p6/Z, [x24, #2, MUL VL]\n" + "ld1w { z17.s }, p0/Z, [x25, #-8, MUL VL]\n" + ".inst 0x648abe63 // bfcvtnt z3.h, p7/M, z19.s\n" + ".inst 0x658abe1c // bfcvt z28.h, p7/M, z16.s\n" + "ld1w { z19.s }, p6/Z, [x24, #2, MUL VL]\n" "whilelt p6.s, XZR, x21\n" "decw x21\n" - ".inst 0x648abe1e // bfcvtnt z30.h, p7/M, z16.s\n" "ld1w { z16.s }, p6/Z, [x25, #-7, MUL VL]\n" - ".inst 0x658abe17 // bfcvt z23.h, p7/M, z16.s\n" - "add x23, x23, %x[out_stride]\n" - "ld1w { z16.s }, p5/Z, [x24, #3, MUL VL]\n" + ".inst 0x648abe42 // bfcvtnt z2.h, p7/M, z18.s\n" + "ld1w { z18.s }, p5/Z, [x24, #3, MUL VL]\n" "whilelt p5.s, XZR, x21\n" "decw x21\n" - ".inst 0x648abe1d // bfcvtnt z29.h, p7/M, z16.s\n" - "ld1w { z16.s }, p5/Z, [x25, #-6, MUL VL]\n" - ".inst 0x658abe16 // bfcvt z22.h, p7/M, z16.s\n" - "ld1w { z16.s }, p4/Z, [x24, #4, MUL VL]\n" + ".inst 0x658abe3b // bfcvt z27.h, p7/M, z17.s\n" + "ld1w { z17.s }, p5/Z, [x25, #-6, MUL VL]\n" + ".inst 0x648abe61 // bfcvtnt z1.h, p7/M, z19.s\n" + "ld1w { z19.s }, p4/Z, [x24, #4, MUL VL]\n" "whilelt p4.s, XZR, x21\n" "decw x21\n" - ".inst 0x648abe1c // bfcvtnt z28.h, p7/M, z16.s\n" + ".inst 0x658abe1a // bfcvt z26.h, p7/M, z16.s\n" "ld1w { z16.s }, p4/Z, [x25, #-5, MUL VL]\n" - ".inst 0x658abe15 // bfcvt z21.h, p7/M, z16.s\n" - "ld1w { z16.s }, p3/Z, [x24, #5, MUL VL]\n" + ".inst 0x648abe40 // bfcvtnt z0.h, p7/M, z18.s\n" + "ld1w { z18.s }, p3/Z, [x24, #5, MUL VL]\n" "whilelt p3.s, XZR, x21\n" "decw x21\n" - ".inst 0x648abe1b // bfcvtnt z27.h, p7/M, z16.s\n" - "ld1w { z16.s }, p3/Z, [x25, #-4, MUL VL]\n" - ".inst 0x658abe14 // bfcvt z20.h, p7/M, z16.s\n" - "ld1w { z16.s }, p2/Z, [x24, #6, MUL VL]\n" + ".inst 0x658abe39 // bfcvt z25.h, p7/M, z17.s\n" + "ld1w { z17.s }, p3/Z, [x25, #-4, MUL VL]\n" + ".inst 0x648abe7f // bfcvtnt z31.h, p7/M, z19.s\n" + "ld1w { z19.s }, p2/Z, [x24, #6, MUL VL]\n" "whilelt p2.s, XZR, x21\n" "decw x21\n" - ".inst 0x648abe1a // bfcvtnt z26.h, p7/M, z16.s\n" + ".inst 0x658abe18 // bfcvt z24.h, p7/M, z16.s\n" "ld1w { z16.s }, p2/Z, [x25, #-3, MUL VL]\n" - ".inst 0x658abe13 // bfcvt z19.h, p7/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x24, #7, MUL VL]\n" + ".inst 0x648abe5e // bfcvtnt z30.h, p7/M, z18.s\n" + "ld1w { z18.s }, p1/Z, [x24, #7, MUL VL]\n" "whilelt p1.s, XZR, x21\n" "decw x21\n" - ".inst 0x648abe19 // bfcvtnt z25.h, p7/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x25, #-2, MUL VL]\n" + ".inst 0x658abe37 // bfcvt z23.h, p7/M, z17.s\n" + "ld1w { z17.s }, p1/Z, [x25, #-2, MUL VL]\n" "addvl x24, x24, #16\n" - ".inst 0x658abe12 // bfcvt z18.h, p7/M, z16.s\n" - "ld1w { z16.s }, p0/Z, [x24, #-8, MUL VL]\n" + ".inst 0x648abe7d // bfcvtnt z29.h, p7/M, z19.s\n" + ".inst 0x658abe16 // bfcvt z22.h, p7/M, z16.s\n" + "ld1w { z19.s }, p0/Z, [x24, #-8, MUL VL]\n" "whilelt p0.s, XZR, x21\n" "cmp x22, #0x0\n" - ".inst 0x648abe18 // bfcvtnt z24.h, p7/M, z16.s\n" "ld1w { z16.s }, p0/Z, [x25, #-1, MUL VL]\n" - ".inst 0x658abe11 // bfcvt z17.h, p7/M, z16.s\n" - "ld1w { z16.s }, p6/Z, [x24, #-7, MUL VL]\n" - ".inst 0x648abe17 // bfcvtnt z23.h, p7/M, z16.s\n" - "ld1w { z16.s }, p5/Z, [x24, #-6, MUL VL]\n" - ".inst 0x648abe16 // bfcvtnt z22.h, p7/M, z16.s\n" + ".inst 0x648abe5c // bfcvtnt z28.h, p7/M, z18.s\n" + ".inst 0x658abe35 // bfcvt z21.h, p7/M, z17.s\n" + "ld1w { z18.s }, p6/Z, [x24, #-7, MUL VL]\n" + "ld1w { z17.s }, p5/Z, [x24, #-6, MUL VL]\n" + ".inst 0x648abe7b // bfcvtnt z27.h, p7/M, z19.s\n" + ".inst 0x658abe14 // bfcvt z20.h, p7/M, z16.s\n" "ld1w { z16.s }, p4/Z, [x24, #-5, MUL VL]\n" - ".inst 0x648abe15 // bfcvtnt z21.h, p7/M, z16.s\n" - "ld1w { z16.s }, p3/Z, [x24, #-4, MUL VL]\n" - ".inst 0x648abe14 // bfcvtnt z20.h, p7/M, z16.s\n" - "ld1w { z16.s }, p2/Z, [x24, #-3, MUL VL]\n" - ".inst 0x648abe13 // bfcvtnt z19.h, p7/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x24, #-2, MUL VL]\n" - ".inst 0x648abe12 // bfcvtnt z18.h, p7/M, z16.s\n" + "ld1w { z19.s }, p3/Z, [x24, #-4, MUL VL]\n" + ".inst 0x648abe5a // bfcvtnt z26.h, p7/M, z18.s\n" + "ld1w { z18.s }, p2/Z, [x24, #-3, MUL VL]\n" + ".inst 0x648abe39 // bfcvtnt z25.h, p7/M, z17.s\n" + "ld1w { z17.s }, p1/Z, [x24, #-2, MUL VL]\n" + ".inst 0x648abe18 // bfcvtnt z24.h, p7/M, z16.s\n" "ld1w { z16.s }, p0/Z, [x24, #-1, MUL VL]\n" - "st1h { z0.h }, p7, [x20]\n" - ".inst 0x648abe11 // bfcvtnt z17.h, p7/M, z16.s\n" - "st1h { z31.h }, p7, [x20, #1, MUL VL]\n" - "st1h { z30.h }, p7, [x20, #2, MUL VL]\n" - "st1h { z29.h }, p7, [x20, #3, MUL VL]\n" - "st1h { z28.h }, p7, [x20, #4, MUL VL]\n" - "st1h { z27.h }, p7, [x20, #5, MUL VL]\n" - "st1h { z26.h }, p7, [x20, #6, MUL VL]\n" - "st1h { z25.h }, p7, [x20, #7, MUL VL]\n" + "st1h { z3.h }, p7, [x20]\n" + ".inst 0x648abe77 // bfcvtnt z23.h, p7/M, z19.s\n" + "st1h { z2.h }, p7, [x20, #1, MUL VL]\n" + ".inst 0x648abe56 // bfcvtnt z22.h, p7/M, z18.s\n" + "st1h { z1.h }, p7, [x20, #2, MUL VL]\n" + ".inst 0x648abe35 // bfcvtnt z21.h, p7/M, z17.s\n" + "st1h { z0.h }, p7, [x20, #3, MUL VL]\n" + ".inst 0x648abe14 // bfcvtnt z20.h, p7/M, z16.s\n" + "st1h { z31.h }, p7, [x20, #4, MUL VL]\n" + "st1h { z30.h }, p7, [x20, #5, MUL VL]\n" + "st1h { z29.h }, p7, [x20, #6, MUL VL]\n" + "st1h { z28.h }, p7, [x20, #7, MUL VL]\n" "addvl x20, x20, #16\n" - "st1h { z24.h }, p7, [x20, #-8, MUL VL]\n" - "st1h { z23.h }, p7, [x20, #-7, MUL VL]\n" - "st1h { z22.h }, p7, [x20, #-6, MUL VL]\n" - "st1h { z21.h }, p7, [x20, #-5, MUL VL]\n" - "st1h { z20.h }, p7, [x20, #-4, MUL VL]\n" - "st1h { z19.h }, p7, [x20, #-3, MUL VL]\n" - "st1h { z18.h }, p7, [x20, #-2, MUL VL]\n" - "st1h { z17.h }, p7, [x20, #-1, MUL VL]\n" + "st1h { z27.h }, p7, [x20, #-8, MUL VL]\n" + "st1h { z26.h }, p7, [x20, #-7, MUL VL]\n" + "st1h { z25.h }, p7, [x20, #-6, MUL VL]\n" + "st1h { z24.h }, p7, [x20, #-5, MUL VL]\n" + "st1h { z23.h }, p7, [x20, #-4, MUL VL]\n" + "st1h { z22.h }, p7, [x20, #-3, MUL VL]\n" + "st1h { z21.h }, p7, [x20, #-2, MUL VL]\n" + "st1h { z20.h }, p7, [x20, #-1, MUL VL]\n" "bgt 2b\n" "3:" // Main row loop: Column loop skip "cmp %x[height], #0x1\n" -- cgit v1.2.1