Update CPU kernel implementations and guard directives

Resolves COMPMID-6023

Change-Id: I868975d14c4f98af6716726feda22405a6a4c891
Signed-off-by: Michael Tyler <michael.tyler@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/9686
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Viet-Hoa Do <viet-hoa.do@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Benchmark: Arm Jenkins <bsgcomp@arm.com>

1 files changed, 62 insertions, 59 deletions

diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp
index b3bde74635..5684f464b6 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp
@@ -29,8 +29,12 @@
 namespace arm_gemm {
 
 void a64_interleaved_bf16fp32_dot_8x12(
-    const bfloat16 *Apanel, const bfloat16 *Bpanel,
-    float *Cpanel, int ablocks, int bblocks, int K) {
+    const bfloat16 *Apanel,
+    const bfloat16 *Bpanel,
+    float *Cpanel,
+    int ablocks,
+    int bblocks,
+    int K) {
 
     struct KernelArgs {
         size_t K = {};
@@ -43,7 +47,6 @@ void a64_interleaved_bf16fp32_dot_8x12(
     ka.bblocks = bblocks;
 
     __asm__ __volatile__(
-
       "1:"  // Height loop
       "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n"
       "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n"
@@ -88,8 +91,8 @@ void a64_interleaved_bf16fp32_dot_8x12(
       "movi v31.16b, #0x0\n"
       "blt 4f\n"
       "3:"  // main loop head
-      "ldr q2, [%x[Apanel], #0x20]\n"
-      "ldr q3, [%x[Apanel], #0x30]\n"
+      "ldr q3, [%x[Apanel], #0x20]\n"
+      "ldr q7, [%x[Apanel], #0x30]\n"
       ".inst 0x4f40f088  // bfdot v8.4s, v4.8h, v0.h[0]\n"
       ".inst 0x4f60f08b  // bfdot v11.4s, v4.8h, v0.h[1]\n"
       ".inst 0x4f40f88e  // bfdot v14.4s, v4.8h, v0.h[2]\n"
@@ -123,35 +126,35 @@ void a64_interleaved_bf16fp32_dot_8x12(
       ".inst 0x4f61f0d9  // bfdot v25.4s, v6.8h, v1.h[1]\n"
       ".inst 0x4f41f8dc  // bfdot v28.4s, v6.8h, v1.h[2]\n"
       ".inst 0x4f61f8df  // bfdot v31.4s, v6.8h, v1.h[3]\n"
-      "ldr q6, [x22, #0x50]\n"
+      "ldr q2, [x22, #0x50]\n"
       "ldr q1, [%x[Apanel], #0x10]\n"
       "add x22, x22, #0x60\n"
-      ".inst 0x4f42f088  // bfdot v8.4s, v4.8h, v2.h[0]\n"
-      ".inst 0x4f62f08b  // bfdot v11.4s, v4.8h, v2.h[1]\n"
-      ".inst 0x4f42f88e  // bfdot v14.4s, v4.8h, v2.h[2]\n"
-      ".inst 0x4f62f891  // bfdot v17.4s, v4.8h, v2.h[3]\n"
-      ".inst 0x4f43f094  // bfdot v20.4s, v4.8h, v3.h[0]\n"
-      ".inst 0x4f63f097  // bfdot v23.4s, v4.8h, v3.h[1]\n"
-      ".inst 0x4f43f89a  // bfdot v26.4s, v4.8h, v3.h[2]\n"
-      ".inst 0x4f63f89d  // bfdot v29.4s, v4.8h, v3.h[3]\n"
+      ".inst 0x4f43f088  // bfdot v8.4s, v4.8h, v3.h[0]\n"
+      ".inst 0x4f63f08b  // bfdot v11.4s, v4.8h, v3.h[1]\n"
+      ".inst 0x4f43f88e  // bfdot v14.4s, v4.8h, v3.h[2]\n"
+      ".inst 0x4f63f891  // bfdot v17.4s, v4.8h, v3.h[3]\n"
+      ".inst 0x4f47f094  // bfdot v20.4s, v4.8h, v7.h[0]\n"
+      ".inst 0x4f67f097  // bfdot v23.4s, v4.8h, v7.h[1]\n"
+      ".inst 0x4f47f89a  // bfdot v26.4s, v4.8h, v7.h[2]\n"
+      ".inst 0x4f67f89d  // bfdot v29.4s, v4.8h, v7.h[3]\n"
       "ldr q4, [x22, #0x0]\n"
-      ".inst 0x4f42f0a9  // bfdot v9.4s, v5.8h, v2.h[0]\n"
-      ".inst 0x4f62f0ac  // bfdot v12.4s, v5.8h, v2.h[1]\n"
-      ".inst 0x4f42f8af  // bfdot v15.4s, v5.8h, v2.h[2]\n"
-      ".inst 0x4f62f8b2  // bfdot v18.4s, v5.8h, v2.h[3]\n"
-      ".inst 0x4f43f0b5  // bfdot v21.4s, v5.8h, v3.h[0]\n"
-      ".inst 0x4f63f0b8  // bfdot v24.4s, v5.8h, v3.h[1]\n"
-      ".inst 0x4f43f8bb  // bfdot v27.4s, v5.8h, v3.h[2]\n"
-      ".inst 0x4f63f8be  // bfdot v30.4s, v5.8h, v3.h[3]\n"
+      ".inst 0x4f43f0a9  // bfdot v9.4s, v5.8h, v3.h[0]\n"
+      ".inst 0x4f63f0ac  // bfdot v12.4s, v5.8h, v3.h[1]\n"
+      ".inst 0x4f43f8af  // bfdot v15.4s, v5.8h, v3.h[2]\n"
+      ".inst 0x4f63f8b2  // bfdot v18.4s, v5.8h, v3.h[3]\n"
+      ".inst 0x4f47f0b5  // bfdot v21.4s, v5.8h, v7.h[0]\n"
+      ".inst 0x4f67f0b8  // bfdot v24.4s, v5.8h, v7.h[1]\n"
+      ".inst 0x4f47f8bb  // bfdot v27.4s, v5.8h, v7.h[2]\n"
+      ".inst 0x4f67f8be  // bfdot v30.4s, v5.8h, v7.h[3]\n"
       "ldr q5, [x22, #0x10]\n"
-      ".inst 0x4f42f0ca  // bfdot v10.4s, v6.8h, v2.h[0]\n"
-      ".inst 0x4f62f0cd  // bfdot v13.4s, v6.8h, v2.h[1]\n"
-      ".inst 0x4f42f8d0  // bfdot v16.4s, v6.8h, v2.h[2]\n"
-      ".inst 0x4f62f8d3  // bfdot v19.4s, v6.8h, v2.h[3]\n"
-      ".inst 0x4f43f0d6  // bfdot v22.4s, v6.8h, v3.h[0]\n"
-      ".inst 0x4f63f0d9  // bfdot v25.4s, v6.8h, v3.h[1]\n"
-      ".inst 0x4f43f8dc  // bfdot v28.4s, v6.8h, v3.h[2]\n"
-      ".inst 0x4f63f8df  // bfdot v31.4s, v6.8h, v3.h[3]\n"
+      ".inst 0x4f43f04a  // bfdot v10.4s, v2.8h, v3.h[0]\n"
+      ".inst 0x4f63f04d  // bfdot v13.4s, v2.8h, v3.h[1]\n"
+      ".inst 0x4f43f850  // bfdot v16.4s, v2.8h, v3.h[2]\n"
+      ".inst 0x4f63f853  // bfdot v19.4s, v2.8h, v3.h[3]\n"
+      ".inst 0x4f47f056  // bfdot v22.4s, v2.8h, v7.h[0]\n"
+      ".inst 0x4f67f059  // bfdot v25.4s, v2.8h, v7.h[1]\n"
+      ".inst 0x4f47f85c  // bfdot v28.4s, v2.8h, v7.h[2]\n"
+      ".inst 0x4f67f85f  // bfdot v31.4s, v2.8h, v7.h[3]\n"
       "ldr q6, [x22, #0x20]\n"
       "bge 3b\n"
       "4:"  // main loop skip
@@ -182,37 +185,37 @@ void a64_interleaved_bf16fp32_dot_8x12(
       ".inst 0x4f41f8dc  // bfdot v28.4s, v6.8h, v1.h[2]\n"
       ".inst 0x4f61f8df  // bfdot v31.4s, v6.8h, v1.h[3]\n"
       "cbz x20, 5f\n"
-      "ldr q0, [%x[Apanel], #0x0]\n"
-      "ldr q1, [%x[Apanel], #0x10]\n"
+      "ldr q4, [%x[Apanel], #0x0]\n"
+      "ldr q3, [%x[Apanel], #0x10]\n"
       "add %x[Apanel], %x[Apanel], #0x20\n"
-      "ldr q7, [x22, #0x0]\n"
-      "ldr q4, [x22, #0x10]\n"
-      ".inst 0x4f40f0e8  // bfdot v8.4s, v7.8h, v0.h[0]\n"
-      "ldr q5, [x22, #0x20]\n"
-      ".inst 0x4f60f0eb  // bfdot v11.4s, v7.8h, v0.h[1]\n"
-      ".inst 0x4f40f8ee  // bfdot v14.4s, v7.8h, v0.h[2]\n"
-      ".inst 0x4f60f8f1  // bfdot v17.4s, v7.8h, v0.h[3]\n"
-      ".inst 0x4f41f0f4  // bfdot v20.4s, v7.8h, v1.h[0]\n"
+      "ldr q2, [x22, #0x0]\n"
+      "ldr q1, [x22, #0x10]\n"
+      ".inst 0x4f44f048  // bfdot v8.4s, v2.8h, v4.h[0]\n"
+      "ldr q0, [x22, #0x20]\n"
+      ".inst 0x4f64f04b  // bfdot v11.4s, v2.8h, v4.h[1]\n"
+      ".inst 0x4f44f84e  // bfdot v14.4s, v2.8h, v4.h[2]\n"
+      ".inst 0x4f64f851  // bfdot v17.4s, v2.8h, v4.h[3]\n"
+      ".inst 0x4f43f054  // bfdot v20.4s, v2.8h, v3.h[0]\n"
       "add x22, x22, #0x30\n"
-      ".inst 0x4f61f0f7  // bfdot v23.4s, v7.8h, v1.h[1]\n"
-      ".inst 0x4f41f8fa  // bfdot v26.4s, v7.8h, v1.h[2]\n"
-      ".inst 0x4f61f8fd  // bfdot v29.4s, v7.8h, v1.h[3]\n"
-      ".inst 0x4f40f089  // bfdot v9.4s, v4.8h, v0.h[0]\n"
-      ".inst 0x4f60f08c  // bfdot v12.4s, v4.8h, v0.h[1]\n"
-      ".inst 0x4f40f88f  // bfdot v15.4s, v4.8h, v0.h[2]\n"
-      ".inst 0x4f60f892  // bfdot v18.4s, v4.8h, v0.h[3]\n"
-      ".inst 0x4f41f095  // bfdot v21.4s, v4.8h, v1.h[0]\n"
-      ".inst 0x4f61f098  // bfdot v24.4s, v4.8h, v1.h[1]\n"
-      ".inst 0x4f41f89b  // bfdot v27.4s, v4.8h, v1.h[2]\n"
-      ".inst 0x4f61f89e  // bfdot v30.4s, v4.8h, v1.h[3]\n"
-      ".inst 0x4f40f0aa  // bfdot v10.4s, v5.8h, v0.h[0]\n"
-      ".inst 0x4f60f0ad  // bfdot v13.4s, v5.8h, v0.h[1]\n"
-      ".inst 0x4f40f8b0  // bfdot v16.4s, v5.8h, v0.h[2]\n"
-      ".inst 0x4f60f8b3  // bfdot v19.4s, v5.8h, v0.h[3]\n"
-      ".inst 0x4f41f0b6  // bfdot v22.4s, v5.8h, v1.h[0]\n"
-      ".inst 0x4f61f0b9  // bfdot v25.4s, v5.8h, v1.h[1]\n"
-      ".inst 0x4f41f8bc  // bfdot v28.4s, v5.8h, v1.h[2]\n"
-      ".inst 0x4f61f8bf  // bfdot v31.4s, v5.8h, v1.h[3]\n"
+      ".inst 0x4f63f057  // bfdot v23.4s, v2.8h, v3.h[1]\n"
+      ".inst 0x4f43f85a  // bfdot v26.4s, v2.8h, v3.h[2]\n"
+      ".inst 0x4f63f85d  // bfdot v29.4s, v2.8h, v3.h[3]\n"
+      ".inst 0x4f44f029  // bfdot v9.4s, v1.8h, v4.h[0]\n"
+      ".inst 0x4f64f02c  // bfdot v12.4s, v1.8h, v4.h[1]\n"
+      ".inst 0x4f44f82f  // bfdot v15.4s, v1.8h, v4.h[2]\n"
+      ".inst 0x4f64f832  // bfdot v18.4s, v1.8h, v4.h[3]\n"
+      ".inst 0x4f43f035  // bfdot v21.4s, v1.8h, v3.h[0]\n"
+      ".inst 0x4f63f038  // bfdot v24.4s, v1.8h, v3.h[1]\n"
+      ".inst 0x4f43f83b  // bfdot v27.4s, v1.8h, v3.h[2]\n"
+      ".inst 0x4f63f83e  // bfdot v30.4s, v1.8h, v3.h[3]\n"
+      ".inst 0x4f44f00a  // bfdot v10.4s, v0.8h, v4.h[0]\n"
+      ".inst 0x4f64f00d  // bfdot v13.4s, v0.8h, v4.h[1]\n"
+      ".inst 0x4f44f810  // bfdot v16.4s, v0.8h, v4.h[2]\n"
+      ".inst 0x4f64f813  // bfdot v19.4s, v0.8h, v4.h[3]\n"
+      ".inst 0x4f43f016  // bfdot v22.4s, v0.8h, v3.h[0]\n"
+      ".inst 0x4f63f019  // bfdot v25.4s, v0.8h, v3.h[1]\n"
+      ".inst 0x4f43f81c  // bfdot v28.4s, v0.8h, v3.h[2]\n"
+      ".inst 0x4f63f81f  // bfdot v31.4s, v0.8h, v3.h[3]\n"
       "5:"  // multiply loop done
       "subs x23, x23, #0x1\n"
       "str q8, [%x[Cpanel], #0x0]\n"