Update CPU kernels and add mixed sign GEMM support

 - Add support for mixed sign quantized convolution.
 - Add support for mixed sign dequantized GEMM.
 - Add SME FP16 GEMV kernel.
 - Change SME vector length function to use RDSVL instead of static variable.
 - Add GEMM dilation support internally (not exposed yet).
 - Remove unused "get_default_activation_values" functions.
 - Add SVE fixed format interleaved BF16 DOT kernel.
 - Updates and optimizations to assembly kernels.

Resolves COMPMID-6926

Change-Id: I227f502502611d4cc4111c89e30c53ce94079544
Signed-off-by: Michael Tyler <michael.tyler@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/11570
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Gunes Bayir <gunes.bayir@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Benchmark: Arm Jenkins <bsgcomp@arm.com>

1 files changed, 79 insertions, 79 deletions

diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp
index 2756327815..896288cdda 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022-2023 Arm Limited.
+ * Copyright (c) 2022-2024 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -43,133 +43,133 @@ void sme_transpose_interleave_16VL_2x2_fp32bf16(bfloat16 *out, const float *in,
       "ptrue p7.b\n"
       "1:"  // Main row loop: Head
       "mov x25, %x[in]\n"
-      "add x24, x25, %x[in_stride]\n"
       "cmp %x[height], #0x1\n"
-      "add %x[in], x24, %x[in_stride]\n"
+      "add x24, x25, %x[in_stride]\n"
       "mov x23, %x[out]\n"
+      "add %x[in], x24, %x[in_stride]\n"
       "csel x24, x24, %x[pad_row], GT\n"
       "sub %x[height], %x[height], #0x2\n"
       "mov x22, %x[width]\n"
       "2:"  // Main row loop: Column loop
       "mov x21, x22\n"
+      "mov x20, x23\n"
       "whilelt p1.s, XZR, x21\n"
-      "ld1w { z16.s }, p1/Z, [x25]\n"
-      ".inst 0x658abe00  // bfcvt z0.h, p7/M, z16.s\n"
       "decw x21\n"
       "whilelt p0.s, XZR, x21\n"
-      "ld1w { z16.s }, p0/Z, [x25, #1, MUL VL]\n"
-      ".inst 0x658abe1f  // bfcvt z31.h, p7/M, z16.s\n"
       "decw x21\n"
+      "ld1w { z16.s }, p1/Z, [x25]\n"
       "whilelt p6.s, XZR, x21\n"
-      "ld1w { z16.s }, p6/Z, [x25, #2, MUL VL]\n"
-      ".inst 0x658abe1e  // bfcvt z30.h, p7/M, z16.s\n"
       "decw x21\n"
+      "ld1w { z18.s }, p0/Z, [x25, #1, MUL VL]\n"
       "whilelt p5.s, XZR, x21\n"
-      "ld1w { z16.s }, p5/Z, [x25, #3, MUL VL]\n"
-      ".inst 0x658abe1d  // bfcvt z29.h, p7/M, z16.s\n"
       "decw x21\n"
+      "ld1w { z17.s }, p6/Z, [x25, #2, MUL VL]\n"
       "whilelt p4.s, XZR, x21\n"
-      "ld1w { z16.s }, p4/Z, [x25, #4, MUL VL]\n"
-      ".inst 0x658abe1c  // bfcvt z28.h, p7/M, z16.s\n"
       "decw x21\n"
+      "ld1w { z19.s }, p5/Z, [x25, #3, MUL VL]\n"
+      ".inst 0x658abe03  // bfcvt z3.h, p7/M, z16.s\n"
       "whilelt p3.s, XZR, x21\n"
-      "ld1w { z16.s }, p3/Z, [x25, #5, MUL VL]\n"
-      ".inst 0x658abe1b  // bfcvt z27.h, p7/M, z16.s\n"
       "decw x21\n"
+      "ld1w { z16.s }, p4/Z, [x25, #4, MUL VL]\n"
+      ".inst 0x658abe42  // bfcvt z2.h, p7/M, z18.s\n"
       "whilelt p2.s, XZR, x21\n"
-      "ld1w { z16.s }, p2/Z, [x25, #6, MUL VL]\n"
-      ".inst 0x658abe1a  // bfcvt z26.h, p7/M, z16.s\n"
       "decw x21\n"
-      "ld1w { z16.s }, p1/Z, [x24]\n"
+      "ld1w { z18.s }, p3/Z, [x25, #5, MUL VL]\n"
+      ".inst 0x658abe21  // bfcvt z1.h, p7/M, z17.s\n"
+      "ld1w { z17.s }, p2/Z, [x25, #6, MUL VL]\n"
+      ".inst 0x658abe60  // bfcvt z0.h, p7/M, z19.s\n"
+      "decw x22, ALL, MUL #16\n"
+      "add x23, x23, %x[out_stride]\n"
+      ".inst 0x658abe1f  // bfcvt z31.h, p7/M, z16.s\n"
+      "ld1w { z19.s }, p1/Z, [x24]\n"
       "whilelt p1.s, XZR, x21\n"
-      ".inst 0x648abe00  // bfcvtnt z0.h, p7/M, z16.s\n"
       "decw x21\n"
       "ld1w { z16.s }, p1/Z, [x25, #7, MUL VL]\n"
       "addvl x25, x25, #16\n"
-      ".inst 0x658abe19  // bfcvt z25.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p0/Z, [x24, #1, MUL VL]\n"
+      ".inst 0x658abe5e  // bfcvt z30.h, p7/M, z18.s\n"
+      ".inst 0x658abe3d  // bfcvt z29.h, p7/M, z17.s\n"
+      "ld1w { z18.s }, p0/Z, [x24, #1, MUL VL]\n"
       "whilelt p0.s, XZR, x21\n"
       "decw x21\n"
-      ".inst 0x648abe1f  // bfcvtnt z31.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p0/Z, [x25, #-8, MUL VL]\n"
-      ".inst 0x658abe18  // bfcvt z24.h, p7/M, z16.s\n"
-      "mov x20, x23\n"
-      "decw x22, ALL, MUL #16\n"
-      "ld1w { z16.s }, p6/Z, [x24, #2, MUL VL]\n"
+      "ld1w { z17.s }, p0/Z, [x25, #-8, MUL VL]\n"
+      ".inst 0x648abe63  // bfcvtnt z3.h, p7/M, z19.s\n"
+      ".inst 0x658abe1c  // bfcvt z28.h, p7/M, z16.s\n"
+      "ld1w { z19.s }, p6/Z, [x24, #2, MUL VL]\n"
       "whilelt p6.s, XZR, x21\n"
       "decw x21\n"
-      ".inst 0x648abe1e  // bfcvtnt z30.h, p7/M, z16.s\n"
       "ld1w { z16.s }, p6/Z, [x25, #-7, MUL VL]\n"
-      ".inst 0x658abe17  // bfcvt z23.h, p7/M, z16.s\n"
-      "add x23, x23, %x[out_stride]\n"
-      "ld1w { z16.s }, p5/Z, [x24, #3, MUL VL]\n"
+      ".inst 0x648abe42  // bfcvtnt z2.h, p7/M, z18.s\n"
+      "ld1w { z18.s }, p5/Z, [x24, #3, MUL VL]\n"
       "whilelt p5.s, XZR, x21\n"
       "decw x21\n"
-      ".inst 0x648abe1d  // bfcvtnt z29.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p5/Z, [x25, #-6, MUL VL]\n"
-      ".inst 0x658abe16  // bfcvt z22.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p4/Z, [x24, #4, MUL VL]\n"
+      ".inst 0x658abe3b  // bfcvt z27.h, p7/M, z17.s\n"
+      "ld1w { z17.s }, p5/Z, [x25, #-6, MUL VL]\n"
+      ".inst 0x648abe61  // bfcvtnt z1.h, p7/M, z19.s\n"
+      "ld1w { z19.s }, p4/Z, [x24, #4, MUL VL]\n"
       "whilelt p4.s, XZR, x21\n"
       "decw x21\n"
-      ".inst 0x648abe1c  // bfcvtnt z28.h, p7/M, z16.s\n"
+      ".inst 0x658abe1a  // bfcvt z26.h, p7/M, z16.s\n"
       "ld1w { z16.s }, p4/Z, [x25, #-5, MUL VL]\n"
-      ".inst 0x658abe15  // bfcvt z21.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p3/Z, [x24, #5, MUL VL]\n"
+      ".inst 0x648abe40  // bfcvtnt z0.h, p7/M, z18.s\n"
+      "ld1w { z18.s }, p3/Z, [x24, #5, MUL VL]\n"
       "whilelt p3.s, XZR, x21\n"
       "decw x21\n"
-      ".inst 0x648abe1b  // bfcvtnt z27.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p3/Z, [x25, #-4, MUL VL]\n"
-      ".inst 0x658abe14  // bfcvt z20.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p2/Z, [x24, #6, MUL VL]\n"
+      ".inst 0x658abe39  // bfcvt z25.h, p7/M, z17.s\n"
+      "ld1w { z17.s }, p3/Z, [x25, #-4, MUL VL]\n"
+      ".inst 0x648abe7f  // bfcvtnt z31.h, p7/M, z19.s\n"
+      "ld1w { z19.s }, p2/Z, [x24, #6, MUL VL]\n"
       "whilelt p2.s, XZR, x21\n"
       "decw x21\n"
-      ".inst 0x648abe1a  // bfcvtnt z26.h, p7/M, z16.s\n"
+      ".inst 0x658abe18  // bfcvt z24.h, p7/M, z16.s\n"
       "ld1w { z16.s }, p2/Z, [x25, #-3, MUL VL]\n"
-      ".inst 0x658abe13  // bfcvt z19.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p1/Z, [x24, #7, MUL VL]\n"
+      ".inst 0x648abe5e  // bfcvtnt z30.h, p7/M, z18.s\n"
+      "ld1w { z18.s }, p1/Z, [x24, #7, MUL VL]\n"
       "whilelt p1.s, XZR, x21\n"
       "decw x21\n"
-      ".inst 0x648abe19  // bfcvtnt z25.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p1/Z, [x25, #-2, MUL VL]\n"
+      ".inst 0x658abe37  // bfcvt z23.h, p7/M, z17.s\n"
+      "ld1w { z17.s }, p1/Z, [x25, #-2, MUL VL]\n"
       "addvl x24, x24, #16\n"
-      ".inst 0x658abe12  // bfcvt z18.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p0/Z, [x24, #-8, MUL VL]\n"
+      ".inst 0x648abe7d  // bfcvtnt z29.h, p7/M, z19.s\n"
+      ".inst 0x658abe16  // bfcvt z22.h, p7/M, z16.s\n"
+      "ld1w { z19.s }, p0/Z, [x24, #-8, MUL VL]\n"
       "whilelt p0.s, XZR, x21\n"
       "cmp x22, #0x0\n"
-      ".inst 0x648abe18  // bfcvtnt z24.h, p7/M, z16.s\n"
       "ld1w { z16.s }, p0/Z, [x25, #-1, MUL VL]\n"
-      ".inst 0x658abe11  // bfcvt z17.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p6/Z, [x24, #-7, MUL VL]\n"
-      ".inst 0x648abe17  // bfcvtnt z23.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p5/Z, [x24, #-6, MUL VL]\n"
-      ".inst 0x648abe16  // bfcvtnt z22.h, p7/M, z16.s\n"
+      ".inst 0x648abe5c  // bfcvtnt z28.h, p7/M, z18.s\n"
+      ".inst 0x658abe35  // bfcvt z21.h, p7/M, z17.s\n"
+      "ld1w { z18.s }, p6/Z, [x24, #-7, MUL VL]\n"
+      "ld1w { z17.s }, p5/Z, [x24, #-6, MUL VL]\n"
+      ".inst 0x648abe7b  // bfcvtnt z27.h, p7/M, z19.s\n"
+      ".inst 0x658abe14  // bfcvt z20.h, p7/M, z16.s\n"
       "ld1w { z16.s }, p4/Z, [x24, #-5, MUL VL]\n"
-      ".inst 0x648abe15  // bfcvtnt z21.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p3/Z, [x24, #-4, MUL VL]\n"
-      ".inst 0x648abe14  // bfcvtnt z20.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p2/Z, [x24, #-3, MUL VL]\n"
-      ".inst 0x648abe13  // bfcvtnt z19.h, p7/M, z16.s\n"
-      "ld1w { z16.s }, p1/Z, [x24, #-2, MUL VL]\n"
-      ".inst 0x648abe12  // bfcvtnt z18.h, p7/M, z16.s\n"
+      "ld1w { z19.s }, p3/Z, [x24, #-4, MUL VL]\n"
+      ".inst 0x648abe5a  // bfcvtnt z26.h, p7/M, z18.s\n"
+      "ld1w { z18.s }, p2/Z, [x24, #-3, MUL VL]\n"
+      ".inst 0x648abe39  // bfcvtnt z25.h, p7/M, z17.s\n"
+      "ld1w { z17.s }, p1/Z, [x24, #-2, MUL VL]\n"
+      ".inst 0x648abe18  // bfcvtnt z24.h, p7/M, z16.s\n"
       "ld1w { z16.s }, p0/Z, [x24, #-1, MUL VL]\n"
-      "st1h { z0.h }, p7, [x20]\n"
-      ".inst 0x648abe11  // bfcvtnt z17.h, p7/M, z16.s\n"
-      "st1h { z31.h }, p7, [x20, #1, MUL VL]\n"
-      "st1h { z30.h }, p7, [x20, #2, MUL VL]\n"
-      "st1h { z29.h }, p7, [x20, #3, MUL VL]\n"
-      "st1h { z28.h }, p7, [x20, #4, MUL VL]\n"
-      "st1h { z27.h }, p7, [x20, #5, MUL VL]\n"
-      "st1h { z26.h }, p7, [x20, #6, MUL VL]\n"
-      "st1h { z25.h }, p7, [x20, #7, MUL VL]\n"
+      "st1h { z3.h }, p7, [x20]\n"
+      ".inst 0x648abe77  // bfcvtnt z23.h, p7/M, z19.s\n"
+      "st1h { z2.h }, p7, [x20, #1, MUL VL]\n"
+      ".inst 0x648abe56  // bfcvtnt z22.h, p7/M, z18.s\n"
+      "st1h { z1.h }, p7, [x20, #2, MUL VL]\n"
+      ".inst 0x648abe35  // bfcvtnt z21.h, p7/M, z17.s\n"
+      "st1h { z0.h }, p7, [x20, #3, MUL VL]\n"
+      ".inst 0x648abe14  // bfcvtnt z20.h, p7/M, z16.s\n"
+      "st1h { z31.h }, p7, [x20, #4, MUL VL]\n"
+      "st1h { z30.h }, p7, [x20, #5, MUL VL]\n"
+      "st1h { z29.h }, p7, [x20, #6, MUL VL]\n"
+      "st1h { z28.h }, p7, [x20, #7, MUL VL]\n"
       "addvl x20, x20, #16\n"
-      "st1h { z24.h }, p7, [x20, #-8, MUL VL]\n"
-      "st1h { z23.h }, p7, [x20, #-7, MUL VL]\n"
-      "st1h { z22.h }, p7, [x20, #-6, MUL VL]\n"
-      "st1h { z21.h }, p7, [x20, #-5, MUL VL]\n"
-      "st1h { z20.h }, p7, [x20, #-4, MUL VL]\n"
-      "st1h { z19.h }, p7, [x20, #-3, MUL VL]\n"
-      "st1h { z18.h }, p7, [x20, #-2, MUL VL]\n"
-      "st1h { z17.h }, p7, [x20, #-1, MUL VL]\n"
+      "st1h { z27.h }, p7, [x20, #-8, MUL VL]\n"
+      "st1h { z26.h }, p7, [x20, #-7, MUL VL]\n"
+      "st1h { z25.h }, p7, [x20, #-6, MUL VL]\n"
+      "st1h { z24.h }, p7, [x20, #-5, MUL VL]\n"
+      "st1h { z23.h }, p7, [x20, #-4, MUL VL]\n"
+      "st1h { z22.h }, p7, [x20, #-3, MUL VL]\n"
+      "st1h { z21.h }, p7, [x20, #-2, MUL VL]\n"
+      "st1h { z20.h }, p7, [x20, #-1, MUL VL]\n"
       "bgt 2b\n"
       "3:"  // Main row loop: Column loop skip
       "cmp %x[height], #0x1\n"