From aac71e563eeb83cbe925c605455122b13f8c0fb4 Mon Sep 17 00:00:00 2001
From: Georgios Pinitas <georgios.pinitas@arm.com>
Date: Thu, 25 Feb 2021 18:57:41 +0000
Subject: Update int8 A55 kernels

Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Change-Id: I15a443806a9773bd18b212fe23d9e4d126948be4
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/5191
Reviewed-by: Michele Di Giorgio <michele.digiorgio@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
---
 .../kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp       | 392 ++++++++++-----------
 1 file changed, 196 insertions(+), 196 deletions(-)

(limited to 'src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp')

diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp
index 690085cb3b..954e2891fb 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp
@@ -130,11 +130,11 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "mov v5.d[1], x24\n"
       "ldr d7, [x10, #0x30]\n"
       "ldr x19, [x10, #0x38]\n"
-      "mov v6.d[1], x23\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
+      "mov v6.d[1], x23\n"
       "ldr d8, [x10, #0x40]\n"
-      "mov v7.d[1], x19\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
+      "mov v7.d[1], x19\n"
       "ldr x23, [x10, #0x48]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
       "ldr d9, [x10, #0x50]\n"
@@ -142,11 +142,11 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "mov v8.d[1], x23\n"
       "ldr d10, [x10, #0x60]\n"
       "ldr x23, [x10, #0x68]\n"
-      "mov v9.d[1], x19\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
+      "mov v9.d[1], x19\n"
       "ldr d4, [x10, #0x70]\n"
-      "mov v10.d[1], x23\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
+      "mov v10.d[1], x23\n"
       "ldr x19, [x10, #0x78]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
       "ldr d5, [x10, #0x80]\n"
@@ -154,11 +154,11 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "mov v4.d[1], x19\n"
       "ldr d6, [x10, #0x90]\n"
       "ldr x23, [x10, #0x98]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
+      "mov v5.d[1], x24\n"
       "ldr d7, [x10, #0xa0]\n"
-      "mov v6.d[1], x23\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
+      "mov v6.d[1], x23\n"
       "ldr x19, [x10, #0xa8]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
       "ldr d8, [x10, #0xb0]\n"
@@ -166,11 +166,11 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "mov v7.d[1], x19\n"
       "ldr d9, [x10, #0xc0]\n"
       "ldr x19, [x10, #0xc8]\n"
-      "mov v8.d[1], x23\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
+      "mov v8.d[1], x23\n"
       "ldr d10, [x10, #0xd0]\n"
-      "mov v9.d[1], x19\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
+      "mov v9.d[1], x19\n"
       "ldr x23, [x10, #0xd8]\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
       "ldr d4, [x10, #0xe0]\n"
@@ -179,10 +179,10 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "ldr d5, [x10, #0xf0]\n"
       "ldr x24, [x10, #0xf8]\n"
       "add x10, x10, #0x100\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
-      "mov v5.d[1], x24\n"
+      "mov v4.d[1], x19\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
+      "mov v5.d[1], x24\n"
       ".inst 0x6fa0e8b3  // udot v19.4s, v5.16b, v0.4b[3]\n"
       "tbnz %x[flags], #31, 8f\n"
       ".inst 0x6e8f940b  // udot v11.4s, v0.16b, v15.16b\n"
@@ -208,24 +208,24 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "ldr q10, [x10, #0x60]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
       "ldr q4, [x10, #0x70]\n"
-      "ldr q5, [x10, #0x80]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q5, [x10, #0x80]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
-      "ldr q7, [x10, #0xa0]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
-      "ldr q8, [x10, #0xb0]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
-      "ldr q9, [x10, #0xc0]\n"
-      "ldr q10, [x10, #0xd0]\n"
+      "ldr q8, [x10, #0xb0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      "ldr q4, [x10, #0xe0]\n"
+      "ldr q9, [x10, #0xc0]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
-      "ldr q5, [x10, #0xf0]\n"
+      "ldr q10, [x10, #0xd0]\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
-      "add x10, x10, #0x100\n"
+      "ldr q4, [x10, #0xe0]\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
+      "ldr q5, [x10, #0xf0]\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
       ".inst 0x6fa0e8b3  // udot v19.4s, v5.16b, v0.4b[3]\n"
       "tbnz %x[flags], #31, 10f\n"
@@ -451,83 +451,83 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "blt 39f\n"
       "37:"  // Height 2: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
       "ldr d5, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
+      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
       "ldr x24, [x10, #0x18]\n"
-      "add x22, x22, #0x10\n"
       "ldr d6, [x10, #0x20]\n"
+      "add x25, x25, #0x10\n"
       "ldr x23, [x10, #0x28]\n"
+      "add x22, x22, #0x10\n"
       "mov v5.d[1], x24\n"
       "ldr d7, [x10, #0x30]\n"
       "ldr x19, [x10, #0x38]\n"
-      "mov v6.d[1], x23\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
+      "mov v6.d[1], x23\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
       "ldr d8, [x10, #0x40]\n"
-      "mov v7.d[1], x19\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
+      "mov v7.d[1], x19\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
       "ldr x23, [x10, #0x48]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
       "ldr d9, [x10, #0x50]\n"
-      "mov v8.d[1], x23\n"
+      ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
       "ldr x19, [x10, #0x58]\n"
+      "mov v8.d[1], x23\n"
       "ldr d10, [x10, #0x60]\n"
+      "ldr x23, [x10, #0x68]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
-      ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
       "mov v9.d[1], x19\n"
-      "ldr x23, [x10, #0x68]\n"
+      ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
       "ldr d4, [x10, #0x70]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
-      ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
       "mov v10.d[1], x23\n"
+      ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
       "ldr x19, [x10, #0x78]\n"
-      "ldr d5, [x10, #0x80]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
+      "ldr d5, [x10, #0x80]\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
-      "mov v4.d[1], x19\n"
       "ldr x24, [x10, #0x88]\n"
+      "mov v4.d[1], x19\n"
       "ldr d6, [x10, #0x90]\n"
+      "ldr x23, [x10, #0x98]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
-      ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
       "mov v5.d[1], x24\n"
-      "ldr x23, [x10, #0x98]\n"
+      ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
       "ldr d7, [x10, #0xa0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
-      ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
       "mov v6.d[1], x23\n"
+      ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
       "ldr x19, [x10, #0xa8]\n"
-      "ldr d8, [x10, #0xb0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      "ldr d8, [x10, #0xb0]\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
-      "mov v7.d[1], x19\n"
       "ldr x23, [x10, #0xb8]\n"
+      "mov v7.d[1], x19\n"
       "ldr d9, [x10, #0xc0]\n"
+      "ldr x19, [x10, #0xc8]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
-      ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
       "mov v8.d[1], x23\n"
-      "ldr x19, [x10, #0xc8]\n"
+      ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
       "ldr d10, [x10, #0xd0]\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
-      ".inst 0x6f81e917  // udot v23.4s, v8.16b, v1.4b[2]\n"
       "mov v9.d[1], x19\n"
+      ".inst 0x6f81e917  // udot v23.4s, v8.16b, v1.4b[2]\n"
       "ldr x23, [x10, #0xd8]\n"
-      "ldr d4, [x10, #0xe0]\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
+      "ldr d4, [x10, #0xe0]\n"
       ".inst 0x6fa1e934  // udot v20.4s, v9.16b, v1.4b[3]\n"
-      "mov v10.d[1], x23\n"
       "ldr x19, [x10, #0xe8]\n"
+      "mov v10.d[1], x23\n"
       "ldr d5, [x10, #0xf0]\n"
-      ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
-      ".inst 0x6fa1e955  // udot v21.4s, v10.16b, v1.4b[3]\n"
-      "mov v4.d[1], x19\n"
       "ldr x24, [x10, #0xf8]\n"
       "add x10, x10, #0x100\n"
+      ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
+      "mov v4.d[1], x19\n"
+      ".inst 0x6fa1e955  // udot v21.4s, v10.16b, v1.4b[3]\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
-      ".inst 0x6fa1e896  // udot v22.4s, v4.16b, v1.4b[3]\n"
       "mov v5.d[1], x24\n"
+      ".inst 0x6fa1e896  // udot v22.4s, v4.16b, v1.4b[3]\n"
       ".inst 0x6fa0e8b3  // udot v19.4s, v5.16b, v0.4b[3]\n"
       ".inst 0x6fa1e8b7  // udot v23.4s, v5.16b, v1.4b[3]\n"
       "tbnz %x[flags], #31, 38f\n"
@@ -544,50 +544,50 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "bge 37b\n"
       "39:"  // Height 2: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
       "ldr q5, [x10, #0x10]\n"
-      "sub x26, x26, #0x10\n"
+      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
       "ldr q7, [x10, #0x30]\n"
-      "add x22, x22, #0x10\n"
+      "sub x26, x26, #0x10\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
+      "ldr q8, [x10, #0x40]\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
+      "ldr q9, [x10, #0x50]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
+      "ldr q10, [x10, #0x60]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
+      "ldr q4, [x10, #0x70]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
+      "ldr q5, [x10, #0x80]\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
-      "ldr q9, [x10, #0x50]\n"
-      "ldr q10, [x10, #0x60]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
+      "ldr q8, [x10, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
+      "ldr q9, [x10, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
-      "ldr q4, [x10, #0x70]\n"
-      "ldr q5, [x10, #0x80]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q10, [x10, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
+      "ldr q4, [x10, #0xe0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
+      "ldr q5, [x10, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
-      "ldr q7, [x10, #0xa0]\n"
-      "ldr q8, [x10, #0xb0]\n"
-      "ldr q9, [x10, #0xc0]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
       ".inst 0x6f81e917  // udot v23.4s, v8.16b, v1.4b[2]\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
       ".inst 0x6fa1e934  // udot v20.4s, v9.16b, v1.4b[3]\n"
-      "ldr q10, [x10, #0xd0]\n"
-      "ldr q4, [x10, #0xe0]\n"
-      "ldr q5, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
       ".inst 0x6fa1e955  // udot v21.4s, v10.16b, v1.4b[3]\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
@@ -617,13 +617,13 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "cmp x26, #0x4\n"
       "ldr q8, [x10, #0x20]\n"
       ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
+      "ldr q9, [x10, #0x30]\n"
       ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0f5  // udot v21.4s, v7.16b, v1.4b[0]\n"
       ".inst 0x6f80e112  // udot v18.4s, v8.16b, v0.4b[0]\n"
       ".inst 0x6f81e116  // udot v22.4s, v8.16b, v1.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f80e133  // udot v19.4s, v9.16b, v0.4b[0]\n"
       ".inst 0x6f81e137  // udot v23.4s, v9.16b, v1.4b[0]\n"
       "bge 42b\n"
@@ -648,13 +648,13 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "ldr q4, [x10, #0x10]\n"
       "ldr q5, [x10, #0x20]\n"
       ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6f81e154  // udot v20.4s, v10.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
       ".inst 0x6f81e095  // udot v21.4s, v4.16b, v1.4b[0]\n"
       ".inst 0x6f80e0b2  // udot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x6f81e0b6  // udot v22.4s, v5.16b, v1.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0d3  // udot v19.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0d7  // udot v23.4s, v6.16b, v1.4b[0]\n"
       "48:"  // Height 2: Multiply loop: No odd multiplies
@@ -907,97 +907,97 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "blt 69f\n"
       "67:"  // Height 3: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
       "ldr d5, [x10, #0x10]\n"
+      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
       "ldr x24, [x10, #0x18]\n"
-      "add x25, x25, #0x10\n"
+      ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
       "ldr d6, [x10, #0x20]\n"
-      "add x22, x22, #0x10\n"
       "ldr x23, [x10, #0x28]\n"
-      "add x21, x21, #0x10\n"
+      "add x25, x25, #0x10\n"
       "mov v5.d[1], x24\n"
       "ldr d7, [x10, #0x30]\n"
       "ldr x19, [x10, #0x38]\n"
-      "mov v6.d[1], x23\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
+      "mov v6.d[1], x23\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
+      "ldr d8, [x10, #0x40]\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
       "mov v7.d[1], x19\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
-      ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
-      ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
-      ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
-      ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
-      ".inst 0x6f82e0fb  // udot v27.4s, v7.16b, v2.4b[0]\n"
-      "ldr d8, [x10, #0x40]\n"
       "ldr x23, [x10, #0x48]\n"
+      ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
       "ldr d9, [x10, #0x50]\n"
+      ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
       "ldr x19, [x10, #0x58]\n"
+      ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
       "mov v8.d[1], x23\n"
+      ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
       "ldr d10, [x10, #0x60]\n"
-      "ldr x23, [x10, #0x68]\n"
+      ".inst 0x6f82e0fb  // udot v27.4s, v7.16b, v2.4b[0]\n"
       "mov v9.d[1], x19\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
+      "ldr x23, [x10, #0x68]\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
+      "ldr d4, [x10, #0x70]\n"
       ".inst 0x6fa2e118  // udot v24.4s, v8.16b, v2.4b[1]\n"
-      "mov v10.d[1], x23\n"
+      "ldr x19, [x10, #0x78]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
+      "mov v10.d[1], x23\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
+      "ldr d5, [x10, #0x80]\n"
       ".inst 0x6fa2e139  // udot v25.4s, v9.16b, v2.4b[1]\n"
+      "mov v4.d[1], x19\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
-      ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
-      ".inst 0x6fa2e15a  // udot v26.4s, v10.16b, v2.4b[1]\n"
-      "ldr d4, [x10, #0x70]\n"
-      "ldr x19, [x10, #0x78]\n"
-      "ldr d5, [x10, #0x80]\n"
       "ldr x24, [x10, #0x88]\n"
-      "mov v4.d[1], x19\n"
+      ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
       "ldr d6, [x10, #0x90]\n"
+      ".inst 0x6fa2e15a  // udot v26.4s, v10.16b, v2.4b[1]\n"
       "ldr x23, [x10, #0x98]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
+      "mov v5.d[1], x24\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
+      "ldr d7, [x10, #0xa0]\n"
       ".inst 0x6fa2e09b  // udot v27.4s, v4.16b, v2.4b[1]\n"
       "mov v6.d[1], x23\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
-      ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
-      ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
-      ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
-      ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
-      ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
-      "ldr d7, [x10, #0xa0]\n"
       "ldr x19, [x10, #0xa8]\n"
+      ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
       "ldr d8, [x10, #0xb0]\n"
+      ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
       "ldr x23, [x10, #0xb8]\n"
+      ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
       "mov v7.d[1], x19\n"
+      ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
       "ldr d9, [x10, #0xc0]\n"
-      "ldr x19, [x10, #0xc8]\n"
+      ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
       "mov v8.d[1], x23\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
+      "ldr x19, [x10, #0xc8]\n"
       ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
+      "ldr d10, [x10, #0xd0]\n"
       ".inst 0x6f82e8fa  // udot v26.4s, v7.16b, v2.4b[2]\n"
-      "mov v9.d[1], x19\n"
+      "ldr x23, [x10, #0xd8]\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
+      "mov v9.d[1], x19\n"
       ".inst 0x6f81e917  // udot v23.4s, v8.16b, v1.4b[2]\n"
+      "ldr d4, [x10, #0xe0]\n"
       ".inst 0x6f82e91b  // udot v27.4s, v8.16b, v2.4b[2]\n"
+      "mov v10.d[1], x23\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
-      ".inst 0x6fa1e934  // udot v20.4s, v9.16b, v1.4b[3]\n"
-      ".inst 0x6fa2e938  // udot v24.4s, v9.16b, v2.4b[3]\n"
-      "ldr d10, [x10, #0xd0]\n"
-      "ldr x23, [x10, #0xd8]\n"
-      "ldr d4, [x10, #0xe0]\n"
       "ldr x19, [x10, #0xe8]\n"
-      "mov v10.d[1], x23\n"
+      ".inst 0x6fa1e934  // udot v20.4s, v9.16b, v1.4b[3]\n"
       "ldr d5, [x10, #0xf0]\n"
+      ".inst 0x6fa2e938  // udot v24.4s, v9.16b, v2.4b[3]\n"
       "ldr x24, [x10, #0xf8]\n"
-      "add x10, x10, #0x100\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
+      "mov v4.d[1], x19\n"
       ".inst 0x6fa1e955  // udot v21.4s, v10.16b, v1.4b[3]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6fa2e959  // udot v25.4s, v10.16b, v2.4b[3]\n"
       "mov v5.d[1], x24\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6fa1e896  // udot v22.4s, v4.16b, v1.4b[3]\n"
       ".inst 0x6fa2e89a  // udot v26.4s, v4.16b, v2.4b[3]\n"
       ".inst 0x6fa0e8b3  // udot v19.4s, v5.16b, v0.4b[3]\n"
@@ -1020,51 +1020,55 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "bge 67b\n"
       "69:"  // Height 3: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
       "ldr q5, [x10, #0x10]\n"
+      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "sub x26, x26, #0x10\n"
+      ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "add x25, x25, #0x10\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
+      "ldr q8, [x10, #0x40]\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
+      "ldr q9, [x10, #0x50]\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
+      "ldr q10, [x10, #0x60]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
+      "ldr q4, [x10, #0x70]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
+      "ldr q5, [x10, #0x80]\n"
       ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
+      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f82e0fb  // udot v27.4s, v7.16b, v2.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
-      "ldr q9, [x10, #0x50]\n"
-      "add x22, x22, #0x10\n"
-      "ldr q10, [x10, #0x60]\n"
-      "add x21, x21, #0x10\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6fa2e118  // udot v24.4s, v8.16b, v2.4b[1]\n"
+      "ldr q8, [x10, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
       ".inst 0x6fa2e139  // udot v25.4s, v9.16b, v2.4b[1]\n"
+      "ldr q9, [x10, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
       ".inst 0x6fa2e15a  // udot v26.4s, v10.16b, v2.4b[1]\n"
-      "ldr q4, [x10, #0x70]\n"
-      "ldr q5, [x10, #0x80]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q10, [x10, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
       ".inst 0x6fa2e09b  // udot v27.4s, v4.16b, v2.4b[1]\n"
+      "ldr q4, [x10, #0xe0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
+      "ldr q5, [x10, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
-      "ldr q7, [x10, #0xa0]\n"
-      "ldr q8, [x10, #0xb0]\n"
-      "ldr q9, [x10, #0xc0]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8fa  // udot v26.4s, v7.16b, v2.4b[2]\n"
@@ -1074,10 +1078,6 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
       ".inst 0x6fa1e934  // udot v20.4s, v9.16b, v1.4b[3]\n"
       ".inst 0x6fa2e938  // udot v24.4s, v9.16b, v2.4b[3]\n"
-      "ldr q10, [x10, #0xd0]\n"
-      "ldr q4, [x10, #0xe0]\n"
-      "ldr q5, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
       ".inst 0x6fa1e955  // udot v21.4s, v10.16b, v1.4b[3]\n"
       ".inst 0x6fa2e959  // udot v25.4s, v10.16b, v2.4b[3]\n"
@@ -1114,7 +1114,9 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "cmp x26, #0x4\n"
       "ldr q8, [x10, #0x20]\n"
       ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
+      "ldr q9, [x10, #0x30]\n"
       ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f82e0d8  // udot v24.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
       ".inst 0x6f81e0f5  // udot v21.4s, v7.16b, v1.4b[0]\n"
@@ -1122,8 +1124,6 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6f80e112  // udot v18.4s, v8.16b, v0.4b[0]\n"
       ".inst 0x6f81e116  // udot v22.4s, v8.16b, v1.4b[0]\n"
       ".inst 0x6f82e11a  // udot v26.4s, v8.16b, v2.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f80e133  // udot v19.4s, v9.16b, v0.4b[0]\n"
       ".inst 0x6f81e137  // udot v23.4s, v9.16b, v1.4b[0]\n"
       ".inst 0x6f82e13b  // udot v27.4s, v9.16b, v2.4b[0]\n"
@@ -1153,7 +1153,9 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "ldr q4, [x10, #0x10]\n"
       "ldr q5, [x10, #0x20]\n"
       ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6f81e154  // udot v20.4s, v10.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f82e158  // udot v24.4s, v10.16b, v2.4b[0]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
       ".inst 0x6f81e095  // udot v21.4s, v4.16b, v1.4b[0]\n"
@@ -1161,8 +1163,6 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6f80e0b2  // udot v18.4s, v5.16b, v0.4b[0]\n"
       ".inst 0x6f81e0b6  // udot v22.4s, v5.16b, v1.4b[0]\n"
       ".inst 0x6f82e0ba  // udot v26.4s, v5.16b, v2.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0d3  // udot v19.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0d7  // udot v23.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0db  // udot v27.4s, v6.16b, v2.4b[0]\n"
@@ -1492,110 +1492,110 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "blt 99f\n"
       "97:"  // Height 4: Multiply loop: Main loop head
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      ".inst 0x6f83e09c  // udot v28.4s, v4.16b, v3.4b[0]\n"
       "ldr d5, [x10, #0x10]\n"
-      "add x25, x25, #0x10\n"
+      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
       "ldr x24, [x10, #0x18]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
       "ldr d6, [x10, #0x20]\n"
-      "add x21, x21, #0x10\n"
+      ".inst 0x6f83e09c  // udot v28.4s, v4.16b, v3.4b[0]\n"
       "ldr x23, [x10, #0x28]\n"
-      "add x20, x20, #0x10\n"
       "mov v5.d[1], x24\n"
       "ldr d7, [x10, #0x30]\n"
       "ldr x19, [x10, #0x38]\n"
-      "mov v6.d[1], x23\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
+      "mov v6.d[1], x23\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
+      "ldr d8, [x10, #0x40]\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
       "mov v7.d[1], x19\n"
       ".inst 0x6f83e0bd  // udot v29.4s, v5.16b, v3.4b[0]\n"
+      "ldr x23, [x10, #0x48]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
+      "ldr d9, [x10, #0x50]\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
+      "ldr x19, [x10, #0x58]\n"
       ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
+      "mov v8.d[1], x23\n"
       ".inst 0x6f83e0de  // udot v30.4s, v6.16b, v3.4b[0]\n"
+      "ldr d10, [x10, #0x60]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
+      "mov v9.d[1], x19\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
+      "ldr x23, [x10, #0x68]\n"
       ".inst 0x6f82e0fb  // udot v27.4s, v7.16b, v2.4b[0]\n"
+      "ldr d4, [x10, #0x70]\n"
       ".inst 0x6f83e0ff  // udot v31.4s, v7.16b, v3.4b[0]\n"
-      "ldr d8, [x10, #0x40]\n"
-      "ldr x23, [x10, #0x48]\n"
-      "ldr d9, [x10, #0x50]\n"
-      "ldr x19, [x10, #0x58]\n"
-      "mov v8.d[1], x23\n"
-      "ldr d10, [x10, #0x60]\n"
-      "ldr x23, [x10, #0x68]\n"
-      "mov v9.d[1], x19\n"
+      "ldr x19, [x10, #0x78]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
+      "mov v10.d[1], x23\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
+      "ldr d5, [x10, #0x80]\n"
       ".inst 0x6fa2e118  // udot v24.4s, v8.16b, v2.4b[1]\n"
-      "mov v10.d[1], x23\n"
+      "mov v4.d[1], x19\n"
       ".inst 0x6fa3e11c  // udot v28.4s, v8.16b, v3.4b[1]\n"
+      "ldr x24, [x10, #0x88]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
+      "ldr d6, [x10, #0x90]\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
+      "ldr x23, [x10, #0x98]\n"
       ".inst 0x6fa2e139  // udot v25.4s, v9.16b, v2.4b[1]\n"
+      "mov v5.d[1], x24\n"
       ".inst 0x6fa3e13d  // udot v29.4s, v9.16b, v3.4b[1]\n"
+      "ldr d7, [x10, #0xa0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
+      "mov v6.d[1], x23\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
+      "ldr x19, [x10, #0xa8]\n"
       ".inst 0x6fa2e15a  // udot v26.4s, v10.16b, v2.4b[1]\n"
+      "ldr d8, [x10, #0xb0]\n"
       ".inst 0x6fa3e15e  // udot v30.4s, v10.16b, v3.4b[1]\n"
-      "ldr d4, [x10, #0x70]\n"
-      "ldr x19, [x10, #0x78]\n"
-      "ldr d5, [x10, #0x80]\n"
-      "ldr x24, [x10, #0x88]\n"
-      "mov v4.d[1], x19\n"
-      "ldr d6, [x10, #0x90]\n"
-      "ldr x23, [x10, #0x98]\n"
-      "mov v5.d[1], x24\n"
+      "ldr x23, [x10, #0xb8]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
+      "mov v7.d[1], x19\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
+      "ldr d9, [x10, #0xc0]\n"
       ".inst 0x6fa2e09b  // udot v27.4s, v4.16b, v2.4b[1]\n"
-      "mov v6.d[1], x23\n"
+      "mov v8.d[1], x23\n"
       ".inst 0x6fa3e09f  // udot v31.4s, v4.16b, v3.4b[1]\n"
+      "ldr x19, [x10, #0xc8]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
+      "ldr d10, [x10, #0xd0]\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
+      "ldr x23, [x10, #0xd8]\n"
       ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
+      "mov v9.d[1], x19\n"
       ".inst 0x6f83e8bc  // udot v28.4s, v5.16b, v3.4b[2]\n"
+      "ldr d4, [x10, #0xe0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      "mov v10.d[1], x23\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
+      "ldr x19, [x10, #0xe8]\n"
       ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
+      "ldr d5, [x10, #0xf0]\n"
       ".inst 0x6f83e8dd  // udot v29.4s, v6.16b, v3.4b[2]\n"
-      "ldr d7, [x10, #0xa0]\n"
-      "ldr x19, [x10, #0xa8]\n"
-      "ldr d8, [x10, #0xb0]\n"
-      "ldr x23, [x10, #0xb8]\n"
-      "mov v7.d[1], x19\n"
-      "ldr d9, [x10, #0xc0]\n"
-      "ldr x19, [x10, #0xc8]\n"
-      "mov v8.d[1], x23\n"
+      "ldr x24, [x10, #0xf8]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
+      "mov v4.d[1], x19\n"
       ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f82e8fa  // udot v26.4s, v7.16b, v2.4b[2]\n"
-      "mov v9.d[1], x19\n"
+      "mov v5.d[1], x24\n"
       ".inst 0x6f83e8fe  // udot v30.4s, v7.16b, v3.4b[2]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6f80e913  // udot v19.4s, v8.16b, v0.4b[2]\n"
+      "add x20, x20, #0x10\n"
       ".inst 0x6f81e917  // udot v23.4s, v8.16b, v1.4b[2]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6f82e91b  // udot v27.4s, v8.16b, v2.4b[2]\n"
       ".inst 0x6f83e91f  // udot v31.4s, v8.16b, v3.4b[2]\n"
       ".inst 0x6fa0e930  // udot v16.4s, v9.16b, v0.4b[3]\n"
       ".inst 0x6fa1e934  // udot v20.4s, v9.16b, v1.4b[3]\n"
       ".inst 0x6fa2e938  // udot v24.4s, v9.16b, v2.4b[3]\n"
       ".inst 0x6fa3e93c  // udot v28.4s, v9.16b, v3.4b[3]\n"
-      "ldr d10, [x10, #0xd0]\n"
-      "ldr x23, [x10, #0xd8]\n"
-      "ldr d4, [x10, #0xe0]\n"
-      "ldr x19, [x10, #0xe8]\n"
-      "mov v10.d[1], x23\n"
-      "ldr d5, [x10, #0xf0]\n"
-      "ldr x24, [x10, #0xf8]\n"
-      "add x10, x10, #0x100\n"
-      "mov v4.d[1], x19\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
       ".inst 0x6fa1e955  // udot v21.4s, v10.16b, v1.4b[3]\n"
       ".inst 0x6fa2e959  // udot v25.4s, v10.16b, v2.4b[3]\n"
-      "mov v5.d[1], x24\n"
       ".inst 0x6fa3e95d  // udot v29.4s, v10.16b, v3.4b[3]\n"
       ".inst 0x6fa0e892  // udot v18.4s, v4.16b, v0.4b[3]\n"
       ".inst 0x6fa1e896  // udot v22.4s, v4.16b, v1.4b[3]\n"
@@ -1625,62 +1625,66 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "bge 97b\n"
       "99:"  // Height 4: Multiply loop: Single iteration only
       ".inst 0x6f80e090  // udot v16.4s, v4.16b, v0.4b[0]\n"
-      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
-      ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
-      ".inst 0x6f83e09c  // udot v28.4s, v4.16b, v3.4b[0]\n"
       "ldr q5, [x10, #0x10]\n"
-      "sub x26, x26, #0x10\n"
+      ".inst 0x6f81e094  // udot v20.4s, v4.16b, v1.4b[0]\n"
       "ldr q6, [x10, #0x20]\n"
-      "add x25, x25, #0x10\n"
+      ".inst 0x6f82e098  // udot v24.4s, v4.16b, v2.4b[0]\n"
       "ldr q7, [x10, #0x30]\n"
-      "add x22, x22, #0x10\n"
+      ".inst 0x6f83e09c  // udot v28.4s, v4.16b, v3.4b[0]\n"
+      "ldr q8, [x10, #0x40]\n"
       ".inst 0x6f80e0b1  // udot v17.4s, v5.16b, v0.4b[0]\n"
+      "ldr q9, [x10, #0x50]\n"
       ".inst 0x6f81e0b5  // udot v21.4s, v5.16b, v1.4b[0]\n"
+      "ldr q10, [x10, #0x60]\n"
       ".inst 0x6f82e0b9  // udot v25.4s, v5.16b, v2.4b[0]\n"
+      "ldr q4, [x10, #0x70]\n"
       ".inst 0x6f83e0bd  // udot v29.4s, v5.16b, v3.4b[0]\n"
+      "ldr q5, [x10, #0x80]\n"
       ".inst 0x6f80e0d2  // udot v18.4s, v6.16b, v0.4b[0]\n"
+      "sub x26, x26, #0x10\n"
       ".inst 0x6f81e0d6  // udot v22.4s, v6.16b, v1.4b[0]\n"
+      "add x25, x25, #0x10\n"
       ".inst 0x6f82e0da  // udot v26.4s, v6.16b, v2.4b[0]\n"
+      "add x22, x22, #0x10\n"
       ".inst 0x6f83e0de  // udot v30.4s, v6.16b, v3.4b[0]\n"
+      "ldr q6, [x10, #0x90]\n"
       ".inst 0x6f80e0f3  // udot v19.4s, v7.16b, v0.4b[0]\n"
+      "add x21, x21, #0x10\n"
       ".inst 0x6f81e0f7  // udot v23.4s, v7.16b, v1.4b[0]\n"
+      "add x20, x20, #0x10\n"
       ".inst 0x6f82e0fb  // udot v27.4s, v7.16b, v2.4b[0]\n"
       ".inst 0x6f83e0ff  // udot v31.4s, v7.16b, v3.4b[0]\n"
-      "ldr q8, [x10, #0x40]\n"
-      "add x21, x21, #0x10\n"
-      "ldr q9, [x10, #0x50]\n"
-      "add x20, x20, #0x10\n"
-      "ldr q10, [x10, #0x60]\n"
+      "ldr q7, [x10, #0xa0]\n"
       ".inst 0x6fa0e110  // udot v16.4s, v8.16b, v0.4b[1]\n"
       ".inst 0x6fa1e114  // udot v20.4s, v8.16b, v1.4b[1]\n"
       ".inst 0x6fa2e118  // udot v24.4s, v8.16b, v2.4b[1]\n"
       ".inst 0x6fa3e11c  // udot v28.4s, v8.16b, v3.4b[1]\n"
+      "ldr q8, [x10, #0xb0]\n"
       ".inst 0x6fa0e131  // udot v17.4s, v9.16b, v0.4b[1]\n"
       ".inst 0x6fa1e135  // udot v21.4s, v9.16b, v1.4b[1]\n"
       ".inst 0x6fa2e139  // udot v25.4s, v9.16b, v2.4b[1]\n"
       ".inst 0x6fa3e13d  // udot v29.4s, v9.16b, v3.4b[1]\n"
+      "ldr q9, [x10, #0xc0]\n"
       ".inst 0x6fa0e152  // udot v18.4s, v10.16b, v0.4b[1]\n"
       ".inst 0x6fa1e156  // udot v22.4s, v10.16b, v1.4b[1]\n"
       ".inst 0x6fa2e15a  // udot v26.4s, v10.16b, v2.4b[1]\n"
       ".inst 0x6fa3e15e  // udot v30.4s, v10.16b, v3.4b[1]\n"
-      "ldr q4, [x10, #0x70]\n"
-      "ldr q5, [x10, #0x80]\n"
-      "ldr q6, [x10, #0x90]\n"
+      "ldr q10, [x10, #0xd0]\n"
       ".inst 0x6fa0e093  // udot v19.4s, v4.16b, v0.4b[1]\n"
       ".inst 0x6fa1e097  // udot v23.4s, v4.16b, v1.4b[1]\n"
       ".inst 0x6fa2e09b  // udot v27.4s, v4.16b, v2.4b[1]\n"
       ".inst 0x6fa3e09f  // udot v31.4s, v4.16b, v3.4b[1]\n"
+      "ldr q4, [x10, #0xe0]\n"
       ".inst 0x6f80e8b0  // udot v16.4s, v5.16b, v0.4b[2]\n"
       ".inst 0x6f81e8b4  // udot v20.4s, v5.16b, v1.4b[2]\n"
       ".inst 0x6f82e8b8  // udot v24.4s, v5.16b, v2.4b[2]\n"
       ".inst 0x6f83e8bc  // udot v28.4s, v5.16b, v3.4b[2]\n"
+      "ldr q5, [x10, #0xf0]\n"
       ".inst 0x6f80e8d1  // udot v17.4s, v6.16b, v0.4b[2]\n"
+      "add x10, x10, #0x100\n"
       ".inst 0x6f81e8d5  // udot v21.4s, v6.16b, v1.4b[2]\n"
       ".inst 0x6f82e8d9  // udot v25.4s, v6.16b, v2.4b[2]\n"
       ".inst 0x6f83e8dd  // udot v29.4s, v6.16b, v3.4b[2]\n"
-      "ldr q7, [x10, #0xa0]\n"
-      "ldr q8, [x10, #0xb0]\n"
-      "ldr q9, [x10, #0xc0]\n"
       ".inst 0x6f80e8f2  // udot v18.4s, v7.16b, v0.4b[2]\n"
       ".inst 0x6f81e8f6  // udot v22.4s, v7.16b, v1.4b[2]\n"
       ".inst 0x6f82e8fa  // udot v26.4s, v7.16b, v2.4b[2]\n"
@@ -1693,10 +1697,6 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6fa1e934  // udot v20.4s, v9.16b, v1.4b[3]\n"
       ".inst 0x6fa2e938  // udot v24.4s, v9.16b, v2.4b[3]\n"
       ".inst 0x6fa3e93c  // udot v28.4s, v9.16b, v3.4b[3]\n"
-      "ldr q10, [x10, #0xd0]\n"
-      "ldr q4, [x10, #0xe0]\n"
-      "ldr q5, [x10, #0xf0]\n"
-      "add x10, x10, #0x100\n"
       ".inst 0x6fa0e951  // udot v17.4s, v10.16b, v0.4b[3]\n"
       ".inst 0x6fa1e955  // udot v21.4s, v10.16b, v1.4b[3]\n"
       ".inst 0x6fa2e959  // udot v25.4s, v10.16b, v2.4b[3]\n"
@@ -1740,7 +1740,9 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "cmp x26, #0x4\n"
       "ldr q8, [x10, #0x20]\n"
       ".inst 0x6f80e0d0  // udot v16.4s, v6.16b, v0.4b[0]\n"
+      "ldr q9, [x10, #0x30]\n"
       ".inst 0x6f81e0d4  // udot v20.4s, v6.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f82e0d8  // udot v24.4s, v6.16b, v2.4b[0]\n"
       ".inst 0x6f83e0dc  // udot v28.4s, v6.16b, v3.4b[0]\n"
       ".inst 0x6f80e0f1  // udot v17.4s, v7.16b, v0.4b[0]\n"
@@ -1751,8 +1753,6 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6f81e116  // udot v22.4s, v8.16b, v1.4b[0]\n"
       ".inst 0x6f82e11a  // udot v26.4s, v8.16b, v2.4b[0]\n"
       ".inst 0x6f83e11e  // udot v30.4s, v8.16b, v3.4b[0]\n"
-      "ldr q9, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f80e133  // udot v19.4s, v9.16b, v0.4b[0]\n"
       ".inst 0x6f81e137  // udot v23.4s, v9.16b, v1.4b[0]\n"
       ".inst 0x6f82e13b  // udot v27.4s, v9.16b, v2.4b[0]\n"
@@ -1787,7 +1787,9 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       "ldr q4, [x10, #0x10]\n"
       "ldr q5, [x10, #0x20]\n"
       ".inst 0x6f80e150  // udot v16.4s, v10.16b, v0.4b[0]\n"
+      "ldr q6, [x10, #0x30]\n"
       ".inst 0x6f81e154  // udot v20.4s, v10.16b, v1.4b[0]\n"
+      "add x10, x10, #0x40\n"
       ".inst 0x6f82e158  // udot v24.4s, v10.16b, v2.4b[0]\n"
       ".inst 0x6f83e15c  // udot v28.4s, v10.16b, v3.4b[0]\n"
       ".inst 0x6f80e091  // udot v17.4s, v4.16b, v0.4b[0]\n"
@@ -1798,8 +1800,6 @@ void a64_hybrid_u8qa_dot_4x16_a55 (
       ".inst 0x6f81e0b6  // udot v22.4s, v5.16b, v1.4b[0]\n"
       ".inst 0x6f82e0ba  // udot v26.4s, v5.16b, v2.4b[0]\n"
       ".inst 0x6f83e0be  // udot v30.4s, v5.16b, v3.4b[0]\n"
-      "ldr q6, [x10, #0x30]\n"
-      "add x10, x10, #0x40\n"
       ".inst 0x6f80e0d3  // udot v19.4s, v6.16b, v0.4b[0]\n"
       ".inst 0x6f81e0d7  // udot v23.4s, v6.16b, v1.4b[0]\n"
       ".inst 0x6f82e0db  // udot v27.4s, v6.16b, v2.4b[0]\n"
-- 
cgit v1.2.1